import time
import unittest
from aminer.analysis.EventTypeDetector import EventTypeDetector
from aminer.input.LogAtom import LogAtom
from aminer.parsing.ParserMatch import ParserMatch
from aminer.parsing.MatchElement import MatchElement
from unit.TestBase import TestBase, DummyMatchContext, DummyFixedDataModelElement, DummySequenceModelElement
from aminer.AminerConfig import DEFAULT_PERSISTENCE_PERIOD


class EventTypeDetectorTest(TestBase):
    """Unittests for the EventTypeDetector."""

    match_context = DummyMatchContext(b" pid=25537 uid=2")
    fdme1 = DummyFixedDataModelElement("s11", b" pid=")
    fdme2 = DummyFixedDataModelElement("d1", b"25537")
    seq1 = DummySequenceModelElement("seq", [fdme1, fdme2])
    match_element1 = seq1.get_match_element("", match_context)

    match_context = DummyMatchContext(b"ddd 25538ddd ")
    fdme3 = DummyFixedDataModelElement("s11", b"ddd ")
    fdme4 = DummyFixedDataModelElement("d1", b"25538")
    seq2 = DummySequenceModelElement("seq", [fdme3, fdme4])
    match_element2 = seq2.get_match_element("", match_context)
    match_element3 = fdme3.get_match_element("/seq", match_context)

    def test1receive_atom(self):
        """Test if log atoms are processed correctly."""
        etd = EventTypeDetector(self.aminer_config, [self.stream_printer_event_handler])
        t = round(time.time(), 3)
        log_atom1 = LogAtom(self.match_element1.match_string, ParserMatch(self.match_element1), t, etd)
        log_atom2 = LogAtom(self.match_element2.match_string, ParserMatch(self.match_element2), t, etd)
        log_atom3 = LogAtom(self.match_element3.match_string, ParserMatch(self.match_element3), t, etd)

        # default arguments
        # In this test case multiple log_atoms are received with default values of the EventTypeDetector. target_path_list is empty and all paths are learned dynamically in variable_key_list.
        self.assertTrue(etd.receive_atom(log_atom1))
        self.assertEqual(self.output_stream.getvalue(), "")
        self.assertEqual(etd.num_events, 1)
        self.assertTrue(all(x in [{"/seq", "/seq/s11", "/seq/d1"}] for x in etd.found_keys))
        self.assertTrue(all(sorted(x) in [sorted(["/seq/d1", "/seq/s11", "/seq"])] for x in etd.variable_key_list))
        self.assertEqual([sorted([y for y in x if isinstance(y[0], (int, float))]) + sorted([y for y in x if isinstance(y[0], str)]) for x in etd.values], [[[25537.0], [" pid="], [" pid=25537"]]])
        self.assertEqual(etd.check_variables, [[True, True, True]])
        self.assertEqual(etd.longest_path, ["/seq/s11"])
        self.assertEqual(etd.id_path_list_tuples, [])
        self.assertEqual(etd.current_index, 0)
        self.assertEqual(etd.num_event_lines, [1])

        self.assertTrue(etd.receive_atom(log_atom2))
        self.assertEqual(etd.num_events, 1)
        self.assertTrue(all(x in [{"/seq", "/seq/s11", "/seq/d1"}] for x in etd.found_keys))
        self.assertTrue(all(sorted(x) in [sorted(["/seq/d1", "/seq/s11", "/seq"])] for x in etd.variable_key_list))
        self.assertEqual([sorted([y for y in x if isinstance(y[0], (int, float))]) + sorted([y for y in x if isinstance(y[0], str)]) for x in etd.values], [[[25537.0, 25538.0], [" pid=", "ddd "], [" pid=25537", "ddd 25538"]]])
        self.assertEqual(etd.check_variables, [[True, True, True]])
        self.assertEqual(etd.longest_path, ["/seq/s11"])
        self.assertEqual(etd.id_path_list_tuples, [])
        self.assertEqual(etd.current_index, 0)
        self.assertEqual(etd.num_event_lines, [2])

        self.assertTrue(etd.receive_atom(log_atom3))
        self.assertEqual(etd.num_events, 2)
        self.assertTrue(all(x in [{"/seq", "/seq/s11", "/seq/d1"}, {"/seq/s11"}] for x in etd.found_keys))
        self.assertTrue(all(sorted(x) in [sorted(["/seq/d1", "/seq/s11", "/seq"]), ["/seq/s11"]] for x in etd.variable_key_list))
        self.assertEqual([sorted([y for y in x if isinstance(y[0], (int, float))]) + sorted([y for y in x if isinstance(y[0], str)]) for x in etd.values], [[[25537.0, 25538.0], [" pid=", "ddd "], [" pid=25537", "ddd 25538"]], [["ddd "]]])
        self.assertEqual(etd.check_variables, [[True, True, True], [True]])
        self.assertEqual(etd.longest_path, ["/seq/s11", "/seq/s11"])
        self.assertEqual(etd.id_path_list_tuples, [])
        self.assertEqual(etd.current_index, 1)
        self.assertEqual(etd.num_event_lines, [2, 1])

        # default arguments + save_values=False
        etd = EventTypeDetector(self.aminer_config, [self.stream_printer_event_handler], save_values=False)
        self.assertTrue(etd.receive_atom(log_atom1))
        self.assertEqual(self.output_stream.getvalue(), "")
        self.assertEqual(etd.num_events, 1)
        self.assertTrue(all(x in [{"/seq", "/seq/s11", "/seq/d1"}] for x in etd.found_keys))
        self.assertTrue(all(sorted(x) in [sorted(["/seq/d1", "/seq/s11", "/seq"])] for x in etd.variable_key_list))
        self.assertEqual(etd.values, [])
        self.assertEqual(etd.check_variables, [])
        self.assertEqual(etd.longest_path, ["/seq/s11"])
        self.assertEqual(etd.id_path_list_tuples, [])
        self.assertEqual(etd.current_index, 0)
        self.assertEqual(etd.num_event_lines, [1])

        self.assertTrue(etd.receive_atom(log_atom2))
        self.assertEqual(etd.num_events, 1)
        self.assertTrue(all(x in [{"/seq", "/seq/s11", "/seq/d1"}] for x in etd.found_keys))
        self.assertTrue(all(sorted(x) in [sorted(["/seq/d1", "/seq/s11", "/seq"])] for x in etd.variable_key_list))
        self.assertEqual(etd.values, [])
        self.assertEqual(etd.check_variables, [])
        self.assertEqual(etd.longest_path, ["/seq/s11"])
        self.assertEqual(etd.id_path_list_tuples, [])
        self.assertEqual(etd.current_index, 0)
        self.assertEqual(etd.num_event_lines, [2])

        self.assertTrue(etd.receive_atom(log_atom3))
        self.assertEqual(etd.num_events, 2)
        self.assertTrue(all(x in [{"/seq", "/seq/s11", "/seq/d1"}, {"/seq/s11"}] for x in etd.found_keys))
        self.assertTrue(all(sorted(x) in [sorted(["/seq/d1", "/seq/s11", "/seq"]), ["/seq/s11"]] for x in etd.variable_key_list))
        self.assertEqual(etd.values, [])
        self.assertEqual(etd.check_variables, [])
        self.assertEqual(etd.longest_path, ["/seq/s11", "/seq/s11"])
        self.assertEqual(etd.id_path_list_tuples, [])
        self.assertEqual(etd.current_index, 1)
        self.assertEqual(etd.num_event_lines, [2, 1])

        # target_path_list + save_values=True
        etd = EventTypeDetector(self.aminer_config, [self.stream_printer_event_handler], target_path_list=["/seq/s11"])
        self.assertTrue(etd.receive_atom(log_atom1))
        self.assertEqual(self.output_stream.getvalue(), "")
        self.assertEqual(etd.num_events, 1)
        self.assertTrue(all(x in [{"/seq", "/seq/s11", "/seq/d1"}] for x in etd.found_keys))
        self.assertTrue(all(sorted(x) in [sorted(["/seq/s11"])] for x in etd.variable_key_list))
        self.assertEqual([sorted([y for y in x if isinstance(y[0], (int, float))]) + sorted([y for y in x if isinstance(y[0], str)]) for x in etd.values], [[[" pid="]]])
        self.assertEqual(etd.check_variables, [[True]])
        self.assertEqual(etd.longest_path, ["/seq/s11"])
        self.assertEqual(etd.id_path_list_tuples, [])
        self.assertEqual(etd.current_index, 0)
        self.assertEqual(etd.num_event_lines, [1])

        self.assertTrue(etd.receive_atom(log_atom2))
        self.assertEqual(etd.num_events, 1)
        self.assertTrue(all(x in [{"/seq", "/seq/s11", "/seq/d1"}] for x in etd.found_keys))
        self.assertTrue(all(sorted(x) in [sorted(["/seq/s11"])] for x in etd.variable_key_list))
        self.assertEqual([sorted([y for y in x if isinstance(y[0], (int, float))]) + sorted([y for y in x if isinstance(y[0], str)]) for x in etd.values], [[[" pid=", "ddd "]]])
        self.assertEqual(etd.check_variables, [[True]])
        self.assertEqual(etd.longest_path, ["/seq/s11"])
        self.assertEqual(etd.id_path_list_tuples, [])
        self.assertEqual(etd.current_index, 0)
        self.assertEqual(etd.num_event_lines, [2])

        self.assertTrue(etd.receive_atom(log_atom3))
        self.assertEqual(etd.num_events, 2)
        self.assertTrue(all(x in [{"/seq", "/seq/s11", "/seq/d1"}, {"/seq/s11"}] for x in etd.found_keys))
        self.assertTrue(all(sorted(x) in [sorted(["/seq/s11"]), ["/seq/s11"]] for x in etd.variable_key_list))
        self.assertEqual([sorted([y for y in x if isinstance(y[0], (int, float))]) + sorted([y for y in x if isinstance(y[0], str)]) for x in etd.values], [[[" pid=", "ddd "]], [["ddd "]]])
        self.assertEqual(etd.check_variables, [[True], [True]])
        self.assertEqual(etd.longest_path, ["/seq/s11", "/seq/s11"])
        self.assertEqual(etd.id_path_list_tuples, [])
        self.assertEqual(etd.current_index, 1)
        self.assertEqual(etd.num_event_lines, [2, 1])

        # target_path_list + save_values=False
        etd = EventTypeDetector(self.aminer_config, [self.stream_printer_event_handler], target_path_list=["/seq/s11"], save_values=False)
        self.assertTrue(etd.receive_atom(log_atom1))
        self.assertEqual(self.output_stream.getvalue(), "")
        self.assertEqual(etd.num_events, 1)
        self.assertTrue(all(x in [{"/seq", "/seq/s11", "/seq/d1"}] for x in etd.found_keys))
        self.assertTrue(all(sorted(x) in [sorted(["/seq/s11"])] for x in etd.variable_key_list))
        self.assertEqual(etd.values, [])
        self.assertEqual(etd.check_variables, [])
        self.assertEqual(etd.longest_path, ["/seq/s11"])
        self.assertEqual(etd.id_path_list_tuples, [])
        self.assertEqual(etd.current_index, 0)
        self.assertEqual(etd.num_event_lines, [1])

        self.assertTrue(etd.receive_atom(log_atom2))
        self.assertEqual(etd.num_events, 1)
        self.assertTrue(all(x in [{"/seq", "/seq/s11", "/seq/d1"}] for x in etd.found_keys))
        self.assertTrue(all(sorted(x) in [sorted(["/seq/s11"])] for x in etd.variable_key_list))
        self.assertEqual(etd.values, [])
        self.assertEqual(etd.check_variables, [])
        self.assertEqual(etd.longest_path, ["/seq/s11"])
        self.assertEqual(etd.id_path_list_tuples, [])
        self.assertEqual(etd.current_index, 0)
        self.assertEqual(etd.num_event_lines, [2])

        self.assertTrue(etd.receive_atom(log_atom3))
        self.assertEqual(etd.num_events, 2)
        self.assertTrue(all(x in [{"/seq", "/seq/s11", "/seq/d1"}, {"/seq/s11"}] for x in etd.found_keys))
        self.assertTrue(all(sorted(x) in [sorted(["/seq/s11"]), ["/seq/s11"]] for x in etd.variable_key_list))
        self.assertEqual(etd.values, [])
        self.assertEqual(etd.check_variables, [])
        self.assertEqual(etd.longest_path, ["/seq/s11", "/seq/s11"])
        self.assertEqual(etd.id_path_list_tuples, [])
        self.assertEqual(etd.current_index, 1)
        self.assertEqual(etd.num_event_lines, [2, 1])

        # id_path_list + save_values=True
        etd = EventTypeDetector(self.aminer_config, [self.stream_printer_event_handler], id_path_list=["/seq/s11"])
        self.assertTrue(etd.receive_atom(log_atom1))
        self.assertEqual(self.output_stream.getvalue(), "")
        self.assertEqual(etd.num_events, 1)
        self.assertTrue(all(x in [{"/seq", "/seq/s11", "/seq/d1"}] for x in etd.found_keys))
        self.assertTrue(all(sorted(x) in [sorted(["/seq/d1", "/seq/s11", "/seq"])] for x in etd.variable_key_list))
        self.assertEqual([sorted([y for y in x if isinstance(y[0], (int, float))]) + sorted([y for y in x if isinstance(y[0], str)]) for x in etd.values], [[[25537.0], [" pid="], [" pid=25537"]]])
        self.assertEqual(etd.check_variables, [[True, True, True]])
        self.assertEqual(etd.longest_path, [])
        self.assertEqual(etd.id_path_list_tuples, [(" pid=",)])
        self.assertEqual(etd.current_index, 0)
        self.assertEqual(etd.num_event_lines, [1])

        self.assertTrue(etd.receive_atom(log_atom2))
        self.assertEqual(etd.num_events, 2)
        self.assertTrue(all(x in [{"/seq", "/seq/s11", "/seq/d1"}] for x in etd.found_keys))
        self.assertTrue(all(sorted(x) in [sorted(["/seq/d1", "/seq/s11", "/seq"])] for x in etd.variable_key_list))
        self.assertEqual([sorted([y for y in x if isinstance(y[0], (int, float))]) + sorted([y for y in x if isinstance(y[0], str)]) for x in etd.values], [[[25537.0], [" pid="], [" pid=25537"]], [[25538.0], ["ddd "], ["ddd 25538"]]])
        self.assertEqual(etd.check_variables, [[True, True, True], [True, True, True]])
        self.assertEqual(etd.longest_path, [])
        self.assertEqual(etd.id_path_list_tuples, [(" pid=",), ("ddd ",)])
        self.assertEqual(etd.current_index, 1)
        self.assertEqual(etd.num_event_lines, [1, 1])

        self.assertTrue(etd.receive_atom(log_atom3))
        self.assertEqual(etd.num_events, 2)
        self.assertTrue(all(x in [{"/seq", "/seq/s11", "/seq/d1"}, {"/seq/s11"}] for x in etd.found_keys))
        self.assertTrue(all(sorted(x) in [sorted(["/seq/d1", "/seq/s11", "/seq"]), ["/seq/s11"]] for x in etd.variable_key_list))
        self.assertEqual([sorted([y for y in x if len(y) > 0 and isinstance(y[0], (int, float))]) + sorted([y for y in x if len(y) > 0 and isinstance(y[0], str)]) + [y for y in x if len(y) == 0] for x in etd.values], [[[25537.0], [" pid="], [" pid=25537"]], [["ddd ", "ddd "], [], []]])
        self.assertEqual(etd.check_variables, [[True, True, True], [y != [] for y in etd.values[1]]])
        self.assertEqual(etd.longest_path, [])
        self.assertEqual(etd.id_path_list_tuples, [(" pid=",), ("ddd ",)])
        self.assertEqual(etd.current_index, 1)
        self.assertEqual(etd.num_event_lines, [1, 2])

        # id_path_list + save_values=False
        etd = EventTypeDetector(self.aminer_config, [self.stream_printer_event_handler], id_path_list=["/seq/s11"], save_values=False)
        self.assertTrue(etd.receive_atom(log_atom1))
        self.assertEqual(self.output_stream.getvalue(), "")
        self.assertEqual(etd.num_events, 1)
        self.assertTrue(all(x in [{"/seq", "/seq/s11", "/seq/d1"}] for x in etd.found_keys))
        self.assertTrue(all(sorted(x) in [sorted(["/seq/d1", "/seq/s11", "/seq"])] for x in etd.variable_key_list))
        self.assertEqual(etd.values, [])
        self.assertEqual(etd.check_variables, [])
        self.assertEqual(etd.longest_path, [])
        self.assertEqual(etd.id_path_list_tuples, [(" pid=",)])
        self.assertEqual(etd.current_index, 0)
        self.assertEqual(etd.num_event_lines, [1])

        self.assertTrue(etd.receive_atom(log_atom2))
        self.assertEqual(etd.num_events, 2)
        self.assertTrue(all(x in [{"/seq", "/seq/s11", "/seq/d1"}] for x in etd.found_keys))
        self.assertTrue(all(sorted(x) in [sorted(["/seq/d1", "/seq/s11", "/seq"])] for x in etd.variable_key_list))
        self.assertEqual(etd.values, [])
        self.assertEqual(etd.check_variables, [])
        self.assertEqual(etd.longest_path, [])
        self.assertEqual(etd.id_path_list_tuples, [(" pid=",), ("ddd ",)])
        self.assertEqual(etd.current_index, 1)
        self.assertEqual(etd.num_event_lines, [1, 1])

        self.assertTrue(etd.receive_atom(log_atom3))
        self.assertEqual(etd.num_events, 2)
        self.assertTrue(all(x in [{"/seq", "/seq/s11", "/seq/d1"}, {"/seq/s11"}] for x in etd.found_keys))
        self.assertTrue(all(sorted(x) in [sorted(["/seq/d1", "/seq/s11", "/seq"]), ["/seq/s11"]] for x in etd.variable_key_list))
        self.assertEqual(etd.values, [])
        self.assertEqual(etd.check_variables, [])
        self.assertEqual(etd.longest_path, [])
        self.assertEqual(etd.id_path_list_tuples, [(" pid=",), ("ddd ",)])
        self.assertEqual(etd.current_index, 1)
        self.assertEqual(etd.num_event_lines, [1, 2])

        # id_path_list + save_values=True + allow_missing_id=True
        etd = EventTypeDetector(self.aminer_config, [self.stream_printer_event_handler], id_path_list=["/seq/s11", "/seq/s12"])
        self.assertFalse(etd.receive_atom(log_atom1))
        self.assertFalse(etd.receive_atom(log_atom2))
        self.assertFalse(etd.receive_atom(log_atom3))

        etd = EventTypeDetector(self.aminer_config, [self.stream_printer_event_handler], id_path_list=["/seq/s11", "/seq/s12"], allow_missing_id=True)
        self.assertTrue(etd.receive_atom(log_atom1))
        self.assertEqual(self.output_stream.getvalue(), "")
        self.assertEqual(etd.num_events, 1)
        self.assertTrue(all(x in [{"/seq", "/seq/s11", "/seq/d1"}] for x in etd.found_keys))
        self.assertTrue(all(sorted(x) in [sorted(["/seq/d1", "/seq/s11", "/seq"])] for x in etd.variable_key_list))
        self.assertEqual([sorted([y for y in x if isinstance(y[0], (int, float))]) + sorted([y for y in x if isinstance(y[0], str)]) for x in etd.values], [[[25537.0], [" pid="], [" pid=25537"]]])
        self.assertEqual(etd.check_variables, [[True, True, True]])
        self.assertEqual(etd.longest_path, [])
        self.assertEqual(etd.id_path_list_tuples, [(" pid=", "")])
        self.assertEqual(etd.current_index, 0)
        self.assertEqual(etd.num_event_lines, [1])

        self.assertTrue(etd.receive_atom(log_atom2))
        self.assertEqual(etd.num_events, 2)
        self.assertTrue(all(x in [{"/seq", "/seq/s11", "/seq/d1"}] for x in etd.found_keys))
        self.assertTrue(all(sorted(x) in [sorted(["/seq/d1", "/seq/s11", "/seq"])] for x in etd.variable_key_list))
        self.assertEqual([sorted([y for y in x if isinstance(y[0], (int, float))]) + sorted([y for y in x if isinstance(y[0], str)]) for x in etd.values], [[[25537.0], [" pid="], [" pid=25537"]], [[25538.0], ["ddd "], ["ddd 25538"]]])
        self.assertEqual(etd.check_variables, [[True, True, True], [True, True, True]])
        self.assertEqual(etd.longest_path, [])
        self.assertEqual(etd.id_path_list_tuples, [(" pid=", ""), ("ddd ", "")])
        self.assertEqual(etd.current_index, 1)
        self.assertEqual(etd.num_event_lines, [1, 1])

        self.assertTrue(etd.receive_atom(log_atom3))
        self.assertEqual(etd.num_events, 2)
        self.assertTrue(all(x in [{"/seq", "/seq/s11", "/seq/d1"}, {"/seq/s11"}] for x in etd.found_keys))
        self.assertTrue(all(sorted(x) in [sorted(["/seq/d1", "/seq/s11", "/seq"]), ["/seq/s11"]] for x in etd.variable_key_list))
        self.assertEqual([sorted([y for y in x if len(y) > 0 and isinstance(y[0], (int, float))]) + sorted([y for y in x if len(y) > 0 and isinstance(y[0], str)]) + [y for y in x if len(y) == 0] for x in etd.values], [[[25537.0], [" pid="], [" pid=25537"]], [["ddd ", "ddd "], [], []]])
        self.assertEqual(etd.check_variables, [[True, True, True], [y != [] for y in etd.values[1]]])
        self.assertEqual(etd.longest_path, [])
        self.assertEqual(etd.id_path_list_tuples, [(" pid=", ""), ("ddd ", "")])
        self.assertEqual(etd.current_index, 1)
        self.assertEqual(etd.num_event_lines, [1, 2])

        # id_path_list + save_values=True + allowed_id_tuples
        etd = EventTypeDetector(self.aminer_config, [self.stream_printer_event_handler], id_path_list=["/seq/s11"], allowed_id_tuples=[(" pid=",)])
        self.assertTrue(etd.receive_atom(log_atom1))
        self.assertEqual(self.output_stream.getvalue(), "")
        self.assertEqual(etd.num_events, 1)
        self.assertTrue(all(x in [{"/seq", "/seq/s11", "/seq/d1"}] for x in etd.found_keys))
        self.assertTrue(all(sorted(x) in [sorted(["/seq/d1", "/seq/s11", "/seq"])] for x in etd.variable_key_list))
        self.assertEqual([sorted([y for y in x if isinstance(y[0], (int, float))]) + sorted([y for y in x if isinstance(y[0], str)]) for x in etd.values], [[[25537.0], [" pid="], [" pid=25537"]]])
        self.assertEqual(etd.check_variables, [[True, True, True]])
        self.assertEqual(etd.longest_path, [])
        self.assertEqual(etd.id_path_list_tuples, [(" pid=",)])
        self.assertEqual(etd.current_index, 0)
        self.assertEqual(etd.num_event_lines, [1])

        self.assertFalse(etd.receive_atom(log_atom2))
        self.assertEqual(etd.num_events, 1)
        self.assertTrue(all(x in [{"/seq", "/seq/s11", "/seq/d1"}] for x in etd.found_keys))
        self.assertTrue(all(sorted(x) in [sorted(["/seq/d1", "/seq/s11", "/seq"])] for x in etd.variable_key_list))
        self.assertEqual([sorted([y for y in x if isinstance(y[0], (int, float))]) + sorted([y for y in x if isinstance(y[0], str)]) for x in etd.values], [[[25537.0], [" pid="], [" pid=25537"]]])
        self.assertEqual(etd.check_variables, [[True, True, True]])
        self.assertEqual(etd.longest_path, [])
        self.assertEqual(etd.id_path_list_tuples, [(" pid=",)])
        self.assertEqual(etd.current_index, -1)
        self.assertEqual(etd.num_event_lines, [1])

        self.assertFalse(etd.receive_atom(log_atom3))
        self.assertEqual(etd.num_events, 1)
        self.assertTrue(all(x in [{"/seq", "/seq/s11", "/seq/d1"}, {"/seq/s11"}] for x in etd.found_keys))
        self.assertTrue(all(sorted(x) in [sorted(["/seq/d1", "/seq/s11", "/seq"]), ["/seq/s11"]] for x in etd.variable_key_list))
        self.assertEqual([sorted([y for y in x if isinstance(y[0], (int, float))]) + sorted([y for y in x if isinstance(y[0], str)]) for x in etd.values], [[[25537.0], [" pid="], [" pid=25537"]]])
        self.assertEqual(etd.check_variables, [[True, True, True]])
        self.assertEqual(etd.longest_path, [])
        self.assertEqual(etd.id_path_list_tuples, [(" pid=",)])
        self.assertEqual(etd.current_index, -1)
        self.assertEqual(etd.num_event_lines, [1])

        # test min_num_vals and max_num_vals
        etd = EventTypeDetector(self.aminer_config, [self.stream_printer_event_handler])
        val_list = [[[]]]
        for i in range(1, etd.max_num_vals + 1, 1):
            log_atom = LogAtom(str(i).encode(), ParserMatch(MatchElement("path", str(i).encode(), i, None)), t, self.__class__.__name__)
            val_list[0][0].append(float(i))
            self.assertTrue(etd.receive_atom(log_atom))
            self.assertEqual(etd.values, val_list)
        i += 1
        log_atom = LogAtom(str(i).encode(), ParserMatch(MatchElement("path", str(i).encode(), i, None)), t, self.__class__.__name__)
        val_list[0][0].append(float(i))
        self.assertTrue(etd.receive_atom(log_atom))
        self.assertEqual(etd.values, [[val_list[0][0][-etd.min_num_vals:]]])

        #, ["/seq/s1", "/seq/d1"]

    def test2do_timer(self):
        """Test if the do_timer method is implemented properly."""
        etd = EventTypeDetector(self.aminer_config, [self.stream_printer_event_handler])
        t = time.time()
        etd.next_persist_time = t + 400
        self.assertEqual(etd.do_timer(t + 200), 200)
        self.assertEqual(etd.do_timer(t + 400), DEFAULT_PERSISTENCE_PERIOD)
        self.assertEqual(etd.do_timer(t + 999), 1)
        self.assertEqual(etd.do_timer(t + 1000), DEFAULT_PERSISTENCE_PERIOD)

    def test3persistence(self):
        """Test the do_persist and load_persistence_data methods."""
        etd = EventTypeDetector(self.aminer_config, [self.stream_printer_event_handler])
        t = round(time.time(), 3)
        log_atom1 = LogAtom(self.match_element1.match_string, ParserMatch(self.match_element1), t, etd)
        log_atom2 = LogAtom(self.match_element2.match_string, ParserMatch(self.match_element2), t, etd)
        log_atom3 = LogAtom(self.match_element3.match_string, ParserMatch(self.match_element3), t, etd)

        etd = EventTypeDetector(self.aminer_config, [self.stream_printer_event_handler], id_path_list=["/seq/s11"])
        self.assertTrue(etd.receive_atom(log_atom1))
        self.assertTrue(etd.receive_atom(log_atom2))
        self.assertTrue(etd.receive_atom(log_atom3))
        self.assertEqual(etd.num_events, 2)
        self.assertTrue(all(x in [{"/seq", "/seq/s11", "/seq/d1"}, {"/seq/s11"}] for x in etd.found_keys))
        self.assertTrue(all(sorted(x) in [sorted(["/seq/d1", "/seq/s11", "/seq"]), ["/seq/s11"]] for x in etd.variable_key_list))
        self.assertEqual([sorted([y for y in x if len(y) > 0 and isinstance(y[0], (int, float))]) + sorted([y for y in x if len(y) > 0 and isinstance(y[0], str)]) + [y for y in x if len(y) == 0] for x in etd.values], [[[25537.0], [" pid="], [" pid=25537"]], [["ddd ", "ddd "], [], []]])
        self.assertEqual(etd.check_variables, [[True, True, True], [y != [] for y in etd.values[1]]])
        self.assertEqual(etd.longest_path, [])
        self.assertEqual(etd.id_path_list_tuples, [(" pid=",), ("ddd ",)])
        self.assertEqual(etd.current_index, 1)
        self.assertEqual(etd.num_event_lines, [1, 2])
        etd.do_persist()
        # with open(etd.persistence_file_name, "r") as f:
        #     self.assertEqual(f.read(), '[[["string:/seq", "string:/seq/s11", "string:/seq/d1"], ["string:/seq", "string:/seq/s11", "string:/seq/d1"]], [["string:/seq", "string:/seq/s11", "string:/seq/d1"], ["string:/seq", "string:/seq/s11", "string:/seq/d1"]], [[["string: pid=25537"], ["string: pid="], [25537.0]], [[], ["string:ddd ", "string:ddd "], []]], [], [[true, true, true], [false, true, false]], [1, 2], [["string: pid="], ["string:ddd "]]]')

        etd.num_events = 0
        etd.found_keys = []
        etd.variable_key_list = []
        etd.values = []
        etd.num_event_lines = []
        etd.current_index = 0
        etd.id_path_list_tuples = []
        etd.load_persistence_data()
        self.assertEqual(etd.num_events, 2)
        self.assertTrue(all(x in [{"/seq", "/seq/s11", "/seq/d1"}, {"/seq/s11"}] for x in etd.found_keys))
        self.assertTrue(all(sorted(x) in [sorted(["/seq/d1", "/seq/s11", "/seq"]), ["/seq/s11"]] for x in etd.variable_key_list))
        self.assertEqual([sorted([y for y in x if len(y) > 0 and isinstance(y[0], (int, float))]) + sorted([y for y in x if len(y) > 0 and isinstance(y[0], str)]) + [y for y in x if len(y) == 0] for x in etd.values], [[[25537.0], [" pid="], [" pid=25537"]], [["ddd ", "ddd "], [], []]])
        self.assertEqual(etd.check_variables, [[True, True, True], [y != [] for y in etd.values[1]]])
        self.assertEqual(etd.longest_path, [])
        self.assertEqual(etd.id_path_list_tuples, [(" pid=",), ("ddd ",)])
        self.assertEqual(etd.current_index, 0)
        self.assertEqual(etd.num_event_lines, [1, 2])

        other = EventTypeDetector(self.aminer_config, [self.stream_printer_event_handler])
        self.assertEqual(etd.num_events, other.num_events)
        self.assertEqual(etd.found_keys, other.found_keys)
        self.assertEqual(etd.variable_key_list, other.variable_key_list)
        self.assertEqual(etd.values, other.values)
        self.assertEqual(etd.check_variables, other.check_variables)
        self.assertEqual(etd.longest_path, other.longest_path)
        self.assertEqual(etd.id_path_list_tuples, other.id_path_list_tuples)
        self.assertEqual(etd.current_index, other.current_index)
        self.assertEqual(etd.num_event_lines, other.num_event_lines)

    def test4validate_parameters(self):
        """Test all initialization parameters for the detector. Input parameters must be validated in the class."""
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, ["default"])
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, None)
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, "")
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, b"Default")
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, True)
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, 123)
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, 123.3)
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, {"id": "Default"})
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, ())
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, set())

        self.assertRaises(ValueError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], persistence_id="")
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], persistence_id=None)
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], persistence_id=b"Default")
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], persistence_id=True)
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], persistence_id=123)
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], persistence_id=123.22)
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], persistence_id={"id": "Default"})
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], persistence_id=["Default"])
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], persistence_id=[])
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], persistence_id=())
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], persistence_id=set())
        EventTypeDetector(self.aminer_config, [self.stream_printer_event_handler], persistence_id="Default")

        self.assertRaises(ValueError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], target_path_list=[""])
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], target_path_list=[b"True"])
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], target_path_list="True")
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], target_path_list=True)
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], target_path_list=123)
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], target_path_list=123.22)
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], target_path_list={"id": "Default"})
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], target_path_list=())
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], target_path_list=set())
        EventTypeDetector(self.aminer_config, [self.stream_printer_event_handler], target_path_list=["default"])
        EventTypeDetector(self.aminer_config, [self.stream_printer_event_handler], target_path_list=None)
        EventTypeDetector(self.aminer_config, [self.stream_printer_event_handler], target_path_list=[])

        self.assertRaises(ValueError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], id_path_list=[""])
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], id_path_list=[b"True"])
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], id_path_list="True")
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], id_path_list=True)
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], id_path_list=123)
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], id_path_list=123.22)
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], id_path_list={"id": "Default"})
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], id_path_list=())
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], id_path_list=set())
        EventTypeDetector(self.aminer_config, [self.stream_printer_event_handler], id_path_list=["default"])
        EventTypeDetector(self.aminer_config, [self.stream_printer_event_handler], id_path_list=None)
        EventTypeDetector(self.aminer_config, [self.stream_printer_event_handler], id_path_list=[])

        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], allow_missing_id=b"True")
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], allow_missing_id="True")
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], allow_missing_id=123)
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], allow_missing_id=123.22)
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], allow_missing_id={"id": "Default"})
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], allow_missing_id=["Default"])
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], allow_missing_id=[])
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], allow_missing_id=())
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], allow_missing_id=set())
        EventTypeDetector(self.aminer_config, [self.stream_printer_event_handler], allow_missing_id=True)

        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], allowed_id_tuples=b"True")
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], allowed_id_tuples="True")
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], allowed_id_tuples=123)
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], allowed_id_tuples=123.22)
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], allowed_id_tuples={"id": "Default"})
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], allowed_id_tuples=["Default"])
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], allowed_id_tuples=[()])
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], allowed_id_tuples=())
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], allowed_id_tuples=set())
        EventTypeDetector(self.aminer_config, [self.stream_printer_event_handler], allowed_id_tuples=[])
        EventTypeDetector(self.aminer_config, [self.stream_printer_event_handler], allowed_id_tuples=None)
        EventTypeDetector(self.aminer_config, [self.stream_printer_event_handler], allowed_id_tuples=[(b"value",)])

        self.assertRaises(ValueError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], min_num_vals=-1)
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], min_num_vals=b"Default")
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], min_num_vals="123")
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], min_num_vals={"id": "Default"})
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], min_num_vals=["Default"])
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], min_num_vals=[])
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], min_num_vals=())
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], min_num_vals=set())
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], min_num_vals=100.22)

        self.assertRaises(ValueError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], max_num_vals=-1)
        self.assertRaises(ValueError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], max_num_vals=0)
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], max_num_vals=b"Default")
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], max_num_vals="123")
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], max_num_vals={"id": "Default"})
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], max_num_vals=["Default"])
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], max_num_vals=[])
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], max_num_vals=())
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], max_num_vals=set())
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], max_num_vals=100.22)

        self.assertRaises(ValueError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], min_num_vals=100, max_num_vals=100)
        self.assertRaises(ValueError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], min_num_vals=101, max_num_vals=100)
        EventTypeDetector(self.aminer_config, [self.stream_printer_event_handler], min_num_vals=10, max_num_vals=100)

        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], save_values=b"True")
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], save_values="True")
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], save_values=123)
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], save_values=123.22)
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], save_values={"id": "Default"})
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], save_values=["Default"])
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], save_values=[])
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], save_values=())
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], save_values=set())
        EventTypeDetector(self.aminer_config, [self.stream_printer_event_handler], save_values=True)

        self.assertRaises(ValueError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], log_resource_ignore_list=["/tmp/syslog"])
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], log_resource_ignore_list="")
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], log_resource_ignore_list=b"Default")
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], log_resource_ignore_list=True)
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], log_resource_ignore_list=123)
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], log_resource_ignore_list=123.22)
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], log_resource_ignore_list={"id": "Default"})
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], log_resource_ignore_list=())
        self.assertRaises(TypeError, EventTypeDetector, self.aminer_config, [self.stream_printer_event_handler], log_resource_ignore_list=set())
        EventTypeDetector(self.aminer_config, [self.stream_printer_event_handler], log_resource_ignore_list=["file:///tmp/syslog"])


if __name__ == "__main__":
    unittest.main()
