#!/usr/bin/python 

# this program evaluates systems that extract temporal information from text 
# tlink -> temporal links

# foreach f (24-a-gold-tlinks/data/ABC19980108.1830.0711.tml); do
# python evaluation_relations/code/temporal_evaluation.py $f $(echo $f | p 's/24-a-gold-tlinks/30-b-trips-relations/g')
# done

# DURING relations are changed to SIMULTANEOUS


import os
import re
import sys
import time

# def get_arg(index):
#     # for arg in sys.argv:
#     return sys.argv[index]


global_prec_matched = 0
global_rec_matched = 0
global_system_total = 0
global_gold_total = 0

import relation_to_timegraph

base_dir = None
debug = None
evaluation_method = None

consider_DURING_as_SIMULTANEOUS = relation_to_timegraph.consider_DURING_as_SIMULTANEOUS


def extract_name(filename):
    parts = re.split('/', filename)
    length = len(parts)
    return parts[length - 1]


def get_directory_path(path):
    name = extract_name(path)
    dir = re.sub(name, '', path)
    if dir == '':
        dir = './'
    return dir


def get_entity_val(word, line):
    if re.search(word + '="[^"]*"', line):
        entity = re.findall(word + '="[^"]*"', line)[0]
        entity = re.sub(word + '=', '', entity)
        entity = re.sub('"', '', entity)
        return entity
    return word


def change_DURING_relation(filetext):
    newtext = ''
    for line in filetext.split('\n'):
        foo = ''
        words = line.split('\t')
        for i in range(0, len(words)):
            if i == 3 and (words[i] == 'DURING' or words[i] == 'DURING_INV'):
                foo += re.sub('DURING', 'SIMULTANEOUS', re.sub('DURING_INV', 'SIMULTANEOUS', words[i])) + '\t'
            else:
                foo += words[i] + '\t'
        newtext += foo.strip() + '\n'
    return newtext


def get_relations(file):
    text = open(file).read()
    newtext = ''
    name = extract_name(file)
    relations = re.findall('<TLINK[^>]*>', text)
    for each in relations:
        core = ''
        ref = ''
        relType = ''
        if re.search('eventInstanceID', each):
            core = get_entity_val('eventInstanceID', each)
        if re.search('timeID', each):
            core = get_entity_val('timeID', each)
        if re.search('relatedToEventInstance', each):
            ref = get_entity_val('relatedToEventInstance', each)
        if re.search('relatedToTime', each):
            ref = get_entity_val('relatedToTime', each)
        if re.search('relType', each):
            relType = get_entity_val('relType', each)
        if core == '' or ref == '' or relType == '':
            print 'MISSING core, ref or relation', each
        else:
            foo = name + '\t' + core + '\t' + ref + '\t' + relType + '\n'
            if debug >= 3:
                print each
                print foo
            newtext += foo + '\n'
    if consider_DURING_as_SIMULTANEOUS == True:
        newtext = change_DURING_relation(newtext)
        # print '$$', newtext
    return newtext


def reverse_relation(rel):
    rel = re.sub('"', '', rel)
    if rel.upper() == 'BEFORE':
        return 'AFTER'
    if rel.upper() == 'AFTER':
        return 'BEFORE'
    if rel.upper() == 'IBEFORE':
        return 'IAFTER'
    if rel.upper() == 'IAFTER':
        return 'IBEFORE'
    if rel.upper() == 'DURING':
        return 'DURING_INV'
    if rel.upper() == 'BEGINS':
        return 'BEGUN_BY'
    if rel.upper() == 'BEGUN_BY':
        return 'BEGINS'
    if rel.upper() == 'ENDS':
        return 'ENDED_BY'
    if rel.upper() == 'ENDED_BY':
        return 'ENDS'
    if rel.upper() == 'INCLUDES':
        return 'IS_INCLUDED'
    if rel.upper() == 'IS_INCLUDED':
        return 'INCLUDES'
    return rel.upper()


def get_triples(tlink_file):
    tlinks = tlink_file  # open(tlink_file).read() # tlink_file #
    relations = ''
    for line in tlinks.split('\n'):
        if line.strip() == '':
            continue
        if debug >= 4:
            print 'sending_triples', line

        words = line.split('\t')
        relations += words[0] + '\t' + words[1] + '\t' + words[2] + '\t' + words[3] + '\n'
        if debug >= 4:
            print 'received_triples', words[0] + '\t' + words[1] + '\t' + words[2] + '\t' + words[3] + '\n'
        if words[1] != words[2]:
            relations += words[0] + '\t' + words[2] + '\t' + words[1] + '\t' + reverse_relation(words[3]) + '\n'
            if debug >= 4:
                print 'received_triples', words[0] + '\t' + words[2] + '\t' + words[1] + '\t' + reverse_relation(
                    words[3]) + '\n'
    return relations


def get_timegraphs(gold, system):
    gold_text = gold  # open(gold).read() # gold #
    system_text = system  # open(system).read() # system #

    tg_gold = relation_to_timegraph.Timegraph()
    tg_gold = relation_to_timegraph.create_timegraph_from_weight_sorted_relations(gold_text, tg_gold)
    tg_gold.final_relations = tg_gold.final_relations + tg_gold.violated_relations
    tg_system = relation_to_timegraph.Timegraph()
    tg_system = relation_to_timegraph.create_timegraph_from_weight_sorted_relations(system_text, tg_system)
    tg_system.final_relations = tg_system.final_relations + tg_system.violated_relations
    return tg_gold, tg_system


# extract entities and relation from tlink line 
def get_x_y_rel(tlinks):
    words = tlinks.split('\t')
    x = words[1]
    y = words[2]
    rel = words[3]
    return x, y, rel


def get_entity_rel(tlink):
    words = tlink.split('\t')
    if len(words) == 3:
        return words[0] + '\t' + words[1] + '\t' + words[2]
    return words[1] + '\t' + words[2] + '\t' + words[3]


def total_relation_matched(A_tlinks, B_tlinks, B_relations, B_tg):
    count = 0
    for tlink in A_tlinks.split('\n'):
        if tlink.strip() == '':
            continue
        if debug >= 2:
            print tlink
        x, y, rel = get_x_y_rel(tlink)
        foo = relation_to_timegraph.interval_rel_X_Y(x, y, B_tg, rel, 'evaluation')
        if re.search(get_entity_rel(tlink.strip()), B_relations):
            count += 1
            if debug >= 2:
                print 'True'
            continue
        if debug >= 2:
            print x, y, rel, foo[1]
        if re.search('true', foo[1]):
            count += 1
    return count


def total_implicit_matched(system_reduced, gold_reduced, gold_tg):
    count = 0
    for tlink in system_reduced.split('\n'):
        if tlink.strip() == '':
            continue
        if debug >= 2:
            print tlink
        if re.search(tlink, gold_reduced):
            continue

        x, y, rel = get_x_y_rel(tlink)
        foo = relation_to_timegraph.interval_rel_X_Y(x, y, gold_tg, rel, 'evaluation')
        if debug >= 2:
            print x, y, rel, foo[1]
        if re.search('true', foo[1]):
            count += 1
    return count


def get_entities(relations):
    included = ''
    for each in relations.split('\n'):
        if each.strip() == '':
            continue
        words = each.split('\t')
        if not re.search('#' + words[1] + '#', included):
            included += '#' + words[1] + '#\n'
        if not re.search('#' + words[2] + '#', included):
            included += '#' + words[2] + '#\n'
    return included


def get_n(relations):
    included = get_entities(relations)
    return (len(included.split('\n')) - 1)


def get_common_n(gold_relations, system_relations):
    gold_entities = get_entities(gold_relations)
    system_entities = get_entities(system_relations)
    common = ''
    for each in gold_entities.split('\n'):
        if each.strip() == '':
            continue
        if re.search(each, system_entities):
            common += each + '\n'
    if debug >= 3:
        print len(gold_entities.split('\n')), len(system_entities.split('\n')), len(common.split('\n'))
        print common.split('\n')
        print gold_entities.split('\n')
    return (len(common.split('\n')) - 1)


def get_ref_minus(gold_relation, system_relations):
    system_entities = get_entities(system_relations)
    count = 0
    for each in gold_relation.split('\n'):
        if each.strip() == '':
            continue
        words = each.split('\t')
        if re.search('#' + words[1] + '#', system_entities) and re.search('#' + words[2] + '#', system_entities):
            count += 1
    return count


def evaluate_two_files_implicit_in_recall(arg1, arg2):
    global global_prec_matched
    global global_rec_matched
    global global_system_total
    global global_gold_total

    if debug >= 1:
        print '\n\n Evaluate', arg1, arg2
    gold_annotation = get_relations(arg1)
    system_annotation = get_relations(arg2)

    tg_gold, tg_system = get_timegraphs(gold_annotation, system_annotation)
    gold_relations = get_triples(gold_annotation)
    system_relations = get_triples(system_annotation)

    # for precision
    if debug >= 2:
        print '\nchecking precision'
    prec_matched = total_relation_matched(tg_system.final_relations, tg_gold.final_relations, gold_relations, tg_gold)
    # for recall 
    if debug >= 2:
        print '\nchecking recall'
    rec_matched = total_relation_matched(tg_gold.final_relations, tg_system.final_relations, system_relations,
                                         tg_system)
    rec_implicit_matched = total_implicit_matched(tg_system.final_relations, tg_gold.final_relations, tg_gold)
    n = get_common_n(tg_gold.final_relations, tg_system.final_relations)

    ##    n = get_n(tg_gold.final_relations)
    ref_plus = 0.5 * n * (n - 1)
    ##    ref_minus = len(tg_gold.final_relations.split('\n'))-1
    ref_minus = rec_matched  ## get_ref_minus(tg_gold.final_relations, tg_system.final_relations)
    w = 0.99 / (1 + ref_plus - ref_minus)  # ref_minus #
    if debug >= 2:
        print 'n =', n
        print 'rec_implicit_matched', rec_implicit_matched
        print 'n, ref_plus, ref_minus', n, ref_plus, ref_minus
        print 'w', w
        print 'rec_matched', rec_matched
        print 'total', (len(tg_gold.final_relations.split('\n')) - 1)

        print 'w*rec_implicit_matched', w * rec_implicit_matched

    if debug >= 2:
        print 'precision', prec_matched, len(tg_system.final_relations.split('\n')) - 1
    if len(tg_system.final_relations.split('\n')) <= 1:
        precision = 0
    else:
        precision = prec_matched * 1.0 / (len(tg_system.final_relations.split('\n')) - 1)

    if debug >= 2:
        print 'recall', rec_matched, len(tg_gold.final_relations.split('\n')) - 1
    if len(tg_gold.final_relations.split('\n')) <= 1:
        recall = 0
    else:
        recall2 = (rec_matched) * 1.0 / (len(tg_gold.final_relations.split('\n')) - 1)
        recall = (rec_matched + w * rec_implicit_matched) * 1.0 / (len(tg_gold.final_relations.split('\n')) - 1)
        if debug >= 2:
            print 'recall2', recall2
            print 'recall', recall

    if debug >= 1:
        print precision, recall, get_fscore(precision, recall)

    global_prec_matched += prec_matched
    global_rec_matched += rec_matched + w * rec_implicit_matched
    global_system_total += len(tg_system.final_relations.split('\n')) - 1
    global_gold_total += len(tg_gold.final_relations.split('\n')) - 1

    return tg_system


def evaluate_two_files(arg1, arg2):
    global global_prec_matched
    global global_rec_matched
    global global_system_total
    global global_gold_total

    if debug >= 1:
        print '\n\nEvaluate', arg1, arg2
    gold_annotation = get_relations(arg1)
    system_annotation = get_relations(arg2)

    tg_gold, tg_system = get_timegraphs(gold_annotation, system_annotation)
    gold_relations = get_triples(gold_annotation)
    system_relations = get_triples(system_annotation)

    # for precision
    if debug >= 2:
        print '\nchecking precision'
    prec_matched = total_relation_matched(tg_system.final_relations, tg_gold.final_relations, gold_relations, tg_gold)
    # for recall 
    if debug >= 2:
        print '\nchecking recall'
    rec_matched = total_relation_matched(tg_gold.final_relations, tg_system.final_relations, system_relations,
                                         tg_system)

    if debug >= 2:
        print 'precision', prec_matched, len(tg_system.final_relations.split('\n')) - 1
    if len(tg_system.final_relations.split('\n')) <= 1:
        precision = 0
    else:
        precision = prec_matched * 1.0 / (len(tg_system.final_relations.split('\n')) - 1)

    if debug >= 2:
        print 'recall', rec_matched, len(tg_gold.final_relations.split('\n')) - 1
    if len(tg_gold.final_relations.split('\n')) <= 1:
        recall = 0
    else:
        recall = rec_matched * 1.0 / (len(tg_gold.final_relations.split('\n')) - 1)

    if debug >= 1:
        print precision, recall

    global_prec_matched += prec_matched
    global_rec_matched += rec_matched
    global_system_total += len(tg_system.final_relations.split('\n')) - 1
    global_gold_total += len(tg_gold.final_relations.split('\n')) - 1

    return tg_system


def evaluate_two_files_acl11(arg1, arg2):
    global global_prec_matched
    global global_rec_matched
    global global_system_total
    global global_gold_total

    if debug >= 1:
        print '\n\n Evaluate', arg1, arg2
    gold_annotation = get_relations(arg1)
    system_annotation = get_relations(arg2)

    tg_gold, tg_system = get_timegraphs(gold_annotation, system_annotation)
    if debug >= 2:
        print '\nnonredundant'
        print tg_gold.nonredundant
        print '\nremove from reduce'
        print tg_gold.remove_from_reduce
        print '\nviolated relations'
        print tg_gold.violated_relations
        print '\nfinal relations'
        print tg_gold.final_relations

    gold_relations = get_triples(gold_annotation)
    system_relations = get_triples(system_annotation)

    # for precision
    if debug >= 2:
        print '\nchecking precision'
    ##prec_matched = total_relation_matched(tg_system.final_relations, tg_gold.final_relations, gold_relations, tg_gold)
    prec_matched = total_relation_matched(system_relations, tg_gold.final_relations, gold_relations, tg_gold)
    # for recall
    if debug >= 2:
        print '\nchecking recall'
    ##rec_matched = total_relation_matched(tg_gold.final_relations, tg_system.final_relations, system_relations, tg_system)
    rec_matched = total_relation_matched(gold_relations, tg_system.final_relations, system_relations, tg_system)

    if debug >= 2:
        # print 'precision', prec_matched, len(tg_system.final_relations.split('\n'))-1
        print 'precision', prec_matched, len(system_relations.split('\n')) - 1
    if len(system_relations.split('\n')) <= 1:
        ##if len(tg_system.final_relations.split('\n')) <= 1:
        precision = 0
    else:
        ##precision = prec_matched*1.0/(len(tg_system.final_relations.split('\n'))-1)
        precision = prec_matched * 1.0 / (len(system_relations.split('\n')) - 1)

    if debug >= 2:
        ##print 'recall', rec_matched, len(tg_gold.final_relations.split('\n'))-1
        print 'recall', rec_matched, len(gold_relations.split('\n')) - 1

    ##if len(tg_gold.final_relations.split('\n')) <= 1:
    if len(gold_relations.split('\n')) <= 1:
        recall = 0
    else:
        # recall = rec_matched*1.0/(len(tg_gold.final_relations.split('\n'))-1)
        recall = rec_matched * 1.0 / (len(gold_relations.split('\n')) - 1)

    if debug >= 1:
        print precision, recall

    global_prec_matched += prec_matched
    global_rec_matched += rec_matched
    ##global_system_total += len(tg_system.final_relations.split('\n'))-1
    global_system_total += len(system_relations.split('\n')) - 1
    ##global_gold_total += len(tg_gold.final_relations.split('\n'))-1
    global_gold_total += len(gold_relations.split('\n')) - 1

    return tg_system


count_relation = 0
count_node = 0
count_chains = 0
count_time = 0


def evaluate_two_folders(gold, system):
    global count_relation
    global count_node
    global count_chains
    global count_time
    count = 0
    if gold[-1] != '/':
        gold += '/'
    if system[-1] != '/':
        system += '/'
    for file in os.listdir(gold):
        if os.path.isdir(gold + file):
            subdir = file + '/'
            if debug >= 1:
                print 'Traverse files in Directory', gold + subdir
            evaluate_two_folders(gold + subdir, system + subdir)
        else:
            goldfile = gold + file
            systemfile = system + file
            if not re.search('DS_Store', file):
                if debug >= 2:
                    print goldfile, systemfile

                start_time = time.time()
                if evaluation_method == 'acl11':
                    tg = evaluate_two_files_acl11(goldfile, systemfile)
                elif evaluation_method == 'implicit_in_recall':
                    tg = evaluate_two_files_implicit_in_recall(goldfile, systemfile)
                else:
                    tg = evaluate_two_files(goldfile, systemfile)
                end_time = time.time()
                if debug >= 1:
                    print end_time - start_time, ',', tg.count_relation, ',', tg.count_node, ',', tg.next_chain + tg.count_cross_chain
                count_time += end_time - start_time
                count_relation += tg.count_relation
                count_node += tg.count_node
                count_chains += tg.next_chain + tg.count_cross_chain


# print count_time, count_relation, count_node, count_chains
#            if count > 5: 
#                break 
#            count += 1 

def get_fscore(p, r):
    if p + r == 0:
        return 0
    return 2.0 * p * r / (p + r)


def final_score():
    global global_prec_matched
    global global_rec_matched
    global global_system_total
    global global_gold_total

    if global_system_total == 0:
        precision = 0
    else:
        precision = global_prec_matched * 1.0 / global_system_total
    if global_gold_total == 0:
        recall = 0
    else:
        recall = global_rec_matched * 1.0 / global_gold_total

    if precision == 0 and recall == 0:
        fscore = 0
    else:
        fscore = get_fscore(precision, recall)
    print '=== Temporal Awareness Score ==='
    if evaluation_method == 'acl11':
        print 'Evaluated with ACL\'11 score, not taking the reduced graph for relations.'
    elif evaluation_method == 'implicit_in_recall':
        print 'Evaluated considering implicit relations in recall as well'
    else:
        print evaluation_method
    print 'Temporal Score\tF1\tP\tR'
    print '\t\t' + str(100 * round(fscore, 6)) + '\t' + str(100 * round(precision, 6)) + '\t' + str(
        100 * round(recall, 6)) + '\t'
    print 'Overall Temporal Awareness Score (F1 score):', str(100 * round(fscore, 6))
    print ''


# take input from command line and give error messages
# call appropriate functions to evaluate 
def input_and_evaluate(argv):
    global basedir
    global debug
    global evaluation_method

    basedir = re.sub('relation_to_timegraph.py', '', argv[0])
    debug = float(argv[3])

    if len(argv) > 4:
        evaluation_method = argv[4].strip()
    else:
        evaluation_method = ''

    cmd_folder = os.path.dirname(basedir)
    if cmd_folder not in sys.path:
        sys.path.insert(0, cmd_folder)

    invalid = 'false'
    if len(argv) < 3:
        invalid = 'true'
    else:
        arg1 = argv[1]
        arg2 = argv[2]
        global directory_path
        directory_path = get_directory_path(argv[0])

        # both arguments are directories
    if invalid == 'false' and os.path.isdir(arg1) and os.path.isdir(arg2):
        # for each files in gold folder, check the performance of that file in system folder
        if debug >= 2:
            print 'compare files in two folders'
        evaluate_two_folders(arg1, arg2)
    elif invalid == 'false' and os.path.isfile(arg1) and os.path.isfile(arg2):
        # compare the performance between two files
        if debug >= 2:
            print 'compare two files'

        evaluate_two_files(arg1, arg2)
        if evaluation_method == 'acl11':
            tg = evaluate_two_files_acl11(arg1, arg2)
        elif evaluation_method == 'implicit_in_recall':
            tg = evaluate_two_files_implicit_in_recall(arg1, arg2)
        else:
            tg = evaluate_two_files(arg1, arg2)
    else:
        invalid = 'true'
        print 'INVALID INPUT FORMAT'
        print '\nto check the performance of a single file:\n\t python evaluate_entities.py gold_file_path system_file_path\n'
        print 'to check the performace of all files in a gold folder:\n\t python evaluate_entities.py gold_folder_path system_folder_path '

    if invalid == 'false':
        performance = 'get'
        # get_performance()
        final_score()


if __name__ == "__main__":
    input_and_evaluate(sys.argv)


# print count_time, count_relation, count_node, count_chains
