from __future__ import absolute_import
from __future__ import print_function

import atexit
import inspect
import os
import traceback

from threading import Lock

import collections

import math
import signal
import time

import numpy as np
import simplejson
import six
import PIL.Image
import sys

from aetros.JobModel import JobModel
from aetros.client import JobClient
from aetros.const import JOB_STATUS
from aetros.cuda_gpu import CudaNotImplementedException
from aetros.git import Git
from aetros.logger import GeneralLogger
from aetros.utils import git, invalid_json_values, read_config, is_ignored, prepend_signal_handler, raise_sigint, \
    read_parameter_by_path, stop_time, read_home_config, lose_parameters_to_full, extract_parameters, get_logger, \
    is_debug, find_config
from aetros.MonitorThread import MonitoringThread
import subprocess

if not isinstance(sys.stdout, GeneralLogger):
    sys.stdout = GeneralLogger(redirect_to=sys.__stdout__)

if not isinstance(sys.stderr, GeneralLogger):
    sys.stderr = GeneralLogger(redirect_to=sys.__stderr__)

last_exit_code = None
original_exit = sys.exit


def patched_exit(status=None):
    global last_exit_code
    last_exit_code = status
    original_exit(status)

sys.exit = patched_exit


def on_shutdown():
    for job in on_shutdown.started_jobs:
        job.on_shutdown()

on_shutdown.started_jobs = []

atexit.register(on_shutdown)


class StdoutApiException(Exception): pass


def Popen(*args, **kwargs):
    """
    Executes a command using subprocess.Popen and redirects output to AETROS and stdout.
    Parses stdout as well for stdout API calls.

    Use read_line argument to read stdout of command's stdout line by line.
    Use returned process stdin to communicate with the command.

    :return: subprocess.Popen
    """

    read_line = None
    if 'read_line' in kwargs:
        read_line = kwargs['read_line']
        del kwargs['read_line']

    p = subprocess.Popen(*args, **kwargs)
    wait_stdout = None
    wait_stderr = None

    if p.stdout:
        wait_stdout = sys.stdout.attach(p.stdout, read_line=read_line)
    if p.stderr:
        wait_stderr = sys.stderr.attach(p.stderr)

    original_wait = p.wait
    def wait():
        original_wait()

        if wait_stdout:
            wait_stdout()
        if wait_stderr:
            wait_stderr()

    p.wait = wait

    return p


def dict_factory(cursor, row):
    d = {}
    for idx, col in enumerate(cursor.description):
        d[col[0]] = row[idx]
    return d


class EventListener:
    def __init__(self):
        self.events = {}

    def on(self, name, callback):
        if name not in self.events:
            self.events[name] = []

        self.events[name].append(callback)

    def fire(self, name, parameter=None):
        if name in self.events:
            for callback in self.events[name]:
                callback(parameter)


def context():
    """
    Returns a new JobBackend instance which connects to AETROS Trainer
    based on "model" in aetros.yml or (internal: env:AETROS_MODEL_NAME environment variable).

    internal: If env:AETROS_JOB_ID is not defined, it creates a new job.

    Job is ended either by calling JobBackend.done(), JobBackend.fail() or JobBackend.abort().
    If the script ends without calling one of the methods above, JobBackend.stop() is called and exit code defines
    whether it is a fail() or done() result.

    :return: JobBackend
    """
    job = JobBackend()

    offline = False
    if '1' == os.getenv('AETROS_OFFLINE', ''):
        offline = True

    if os.getenv('AETROS_JOB_ID'):
        job.load(os.getenv('AETROS_JOB_ID'))
        if not offline:
            job.connect()
    else:
        job.create()
        if not offline:
            job.connect()

    job.start(offline=offline)

    return job


class JobLossChannel:
    """
    :type job_backend : JobBackend
    """

    def __init__(self, job_backend, name, xaxis=None, yaxis=None, layout=None):
        self.name = name
        self.job_backend = job_backend
        message = {
            'name': self.name,
            'traces': [{'name': 'training'}, {'name': 'validation'}],
            'type': JobChannel.NUMBER,
            'main': True,
            'timed': True,
            'xaxis': xaxis,
            'yaxis': yaxis,
            'layout': layout,
            'lossChannel': True
        }
        self.lock = Lock()

        self.job_backend.git.commit_json_file('CREATE_CHANNEL', 'aetros/job/channel/' + name+ '/config', message)
        self.stream = self.job_backend.git.stream_file('aetros/job/channel/' + name+ '/data.csv')
        self.stream.write('"time", "x","training","validation"\n')

    def send(self, x, training, validation):
        line = simplejson.dumps([self.job_backend.get_run_time(), x, training, validation])[1:-1]

        self.lock.acquire()
        try:
            self.stream.write(line + "\n")
            self.job_backend.git.store_file('aetros/job/channel/' + self.name + '/last.csv', line)
        finally:
            self.lock.release()


class JobImage:
    def __init__(self, name, pil_image, label=None, pos=None):
        self.id = name
        if not isinstance(pil_image, PIL.Image.Image):
            raise Exception('JobImage requires a PIL.Image as image argument.')

        self.image = pil_image
        self.label = label
        self.pos = pos

        if self.pos is None:
            self.pos = time.time()


class JobChannel:
    NUMBER = 'number'
    TEXT = 'text'

    """
    :type job_backend: JobBackend
    """

    def __init__(self, job_backend, name, traces=None,
                 main=False, kpi=False, kpiTrace=0, max_optimization=True,
                 type=None, xaxis=None, yaxis=None, layout=None):
        """
        :param job_backend: JobBakend
        :param name: str
        :param traces: None|list : per default create a trace based on "name".
        :param main: bool : whether this channel is visible in the job list as column for better comparison.

        :param kpi: bool : whether this channel is the KPI (key performance indicator).
                           Used for hyperparameter optimization. Only one channel can be a kpi. Only first trace used.
        :param kpiTrace: bool : if you have multiple traces, define which is the KPI. 0 based index.

        :param max_optimization: bool : whether the optimization maximizes or minmizes the kpi . Use max_optimization=False to
                                        tell the optimization algorithm that his channel minimizes a kpi, for instance the loss of a model.

        :param type: str : One of JobChannel.NUMBER, JobChannel.TEXT, JobChannel.IMAGE
        :param xaxis: dict
        :param yaxis: dict
        :param layout: dict
        """
        self.name = name
        self.job_backend = job_backend
        self.kpi = kpi
        self.kpiTrace = kpiTrace
        self.lock = Lock()

        if self.kpi:
            self.job_backend.kpi_channel = self

        if not (isinstance(traces, list) or traces is None):
            raise Exception(
                'traces can only be None or a list of dicts: [{name: "name", option1: ...}, {name: "name2"}, ...]')

        if not traces:
            traces = [{'name': ''}]

        if isinstance(traces, list) and isinstance(traces[0], six.string_types):
            traces = list(map(lambda x: {'name': x}, traces))

        message = {
            'name': name,
            'traces': traces,
            'type': type or JobChannel.NUMBER,
            'main': main,
            'kpi': kpi,
            'timed': True,
            'kpiTrace': kpiTrace,
            'maxOptimization': max_optimization,
            'xaxis': xaxis,
            'yaxis': yaxis,
            'layout': layout,
        }
        self.traces = traces
        self.job_backend.git.commit_json_file('CREATE_CHANNEL', 'aetros/job/channel/' + name+ '/config', message)
        self.stream = self.job_backend.git.stream_file('aetros/job/channel/' + name+ '/data.csv')

        if self.kpi:
            self.job_backend.git.commit_file('KPI_CHANNEL', 'aetros/job/kpi/name', name)

        line = simplejson.dumps(['time', 'x'] + [str(x['name']) for x in traces])[1:-1]
        self.stream.write(line + "\n")

    def send(self, x, y):
        if not isinstance(y, list):
            y = [y]

        if len(y) != len(self.traces):
            raise Exception(
                'You tried to set more y values (%d items) then traces available in channel %s (%d traces).' % (
                    len(y), self.name, len(self.traces)))

        for v in y:
            if not isinstance(v, (int, float)) and not isinstance(v, six.string_types):
                raise Exception('Could not send channel value for ' + self.name+' since type ' + type(y).__name__+' is not supported. Use int, float or string values.')

        line = simplejson.dumps([self.job_backend.get_run_time(), x] + y)[1:-1]

        self.lock.acquire()
        try:
            self.stream.write(line + "\n")
            self.job_backend.git.store_file('aetros/job/channel/' + self.name + '/last.csv', line)
        finally:
            self.lock.release()

        if self.kpi:
            self.job_backend.git.store_file('aetros/job/kpi/last.json', simplejson.dumps(y[self.kpiTrace]))


class JobBackend:
    """
    :type event_listener: EventListener
    :type id: str: model name
    :type job_id: str: job id
    :type client: Client
    :type job: dict
    """

    def __init__(self, model_name=None, logger=None, config_path='aetros.yml', name=None):
        self.event_listener = EventListener()

        self.log_file_handle = None
        self.job = {'parameters': {}}

        self.git = None

        self.ssh_stream = None
        self.model_name = model_name
        self.logger = logger
        self.config_path = config_path

        self.name = name
        if not self.name:
            if os.getenv('AETROS_JOB_NAME'):
                self.name = os.getenv('AETROS_JOB_NAME')
            else:
                self.name = 'master'

        self.client = None
        self.stream_log = None
        self.speed_stream = None
        self.parameter_change_callback = None
        self.lock = Lock()

        self.last_speed = 0
        self.last_speed_time = 0
        self.last_step_time = time.time()
        self.last_step = 0
        self.made_steps_since_last_sync = 0
        self.made_steps_size_since_last_sync = 0

        self.current_epoch = 0
        self.total_epochs = 0

        self.step_label = None
        self.step_speed_label = None

        # indicates whether early_stop() has been called. Is called by reaching maxTimes or maxEpochs limitation.
        # This flag stops exiting with > 0, since the reach of a limitation is a valid exit.
        self.in_early_stop = False

        # ended means: done, abort or fail method has been called.
        self.ended = False

        # when stop(wait_for_client=True) is called, we sync last messages.
        # this flag indicates that end() hasn't been called yet
        self.stopped = False

        # running means: the syncer client is running.
        self.running = False

        # whether it has started once
        self.started = False
        self.start_time = time.time()

        # whether we are in paused state
        self.is_paused = False

        # whether on_shutdown has been called and thus the python interpreter is dying.
        self.in_shutdown = False

        self.insight_images_info = {}
        self.insight_created = []

        self.monitoring_thread = None

        if not self.logger:
            self.logger = get_logger(os.getenv('AETROS_JOB_NAME', 'aetros-job'))

        self.last_progress_call = None
        self.job_ids = []
        self.in_request = False
        self.stop_requested = False
        self.stop_requested_force = False

        self.kpi_channel = None
        self.progresses = {}

        self.event_listener.on('stop', self.external_stop)
        self.event_listener.on('aborted', self.external_aborted)
        self.event_listener.on('registration', self.on_registration)
        self.event_listener.on('registration_failed', self.on_registration_failed)
        self.event_listener.on('offline', self.on_client_offline)
        self.event_listener.on('parameter_changed', self.on_parameter_changed)
        self.event_listener.on('action', self.on_action)

        if hasattr(signal, 'SIGUSR1'):
            prepend_signal_handler(signal.SIGUSR1, self.on_signusr1)

        self.pid = os.getpid()

        self.ensure_model_name()
        self.home_config = read_home_config()
        self.client = JobClient(self.home_config, self.event_listener, self.logger)
        self.git = Git(self.logger, self.client, self.home_config, self.model_name, self.is_master_process())

        self.logger.debug("Started tracking of job files in git %s for remote %s" % (self.git.git_path, self.git.origin_url))

    @property
    def log_level(self):
        if os.getenv('DEBUG') == '1':
            return 'DEBUG'

        return 'INFO'

    @property
    def host(self):
        return self.home_config['host']

    def get_run_time(self, precision=3):
        return round(time.time() - self.start_time, precision)

    def section(self, title):
        title = title.replace("\t", "  ")
        seconds = self.get_run_time()
        line = "## %s\t%.2f\n" % (title, seconds)
        sys.stdout.write(line)
        sys.stdout.flush()

    def on_registration_failed(self, params):
        self.logger.warning("Connecting to AETROS Trainer at %s failed. Reasons: %s" % (self.host, params['reason'],))
        if 'Permission denied' in params['reason']:
            self.logger.warning("Make sure you have saved your ssh pub key in your AETROS Trainer user account.")

    def on_client_offline(self, params):
        if self.is_master_process():
            self.logger.warning("Could not establish a connection. We stopped automatic syncing.")
            self.logger.warning("You can publish later this job to AETROS Trainer by executing following command.")
            self.logger.warning("$ aetros job-push " + self.job_id[0:9])

        self.git.online = False

    def on_registration(self, params):
        pass

    def on_signusr1(self, signal, frame):
        self.logger.warning("USR1: backend job_id=%s (running=%s, ended=%s), client (online=%s, active=%s, registered=%s, "
                            "connected=%s, queue=%s), git (active_thread=%s, last_push_time=%s)." % (
          str(self.job_id),
          str(self.running),
          str(self.ended),
          str(self.client.online),
          str(self.client.active),
          str(self.client.registered),
          str(self.client.connected),
          str([str(i)+':'+str(len(x)) for i, x in six.iteritems(self.client.queues)]),
          str(self.git.active_thread),
          str(self.git.last_push_time),
        ))

    def on_force_exit(self):
        """
        External hook.
        """
        pass

    def on_sigint(self, sig, frame):
        """
        We got SIGINT signal.
        """

        if self.stop_requested or self.stop_requested_force:
            # signal has already been sent or we force a shutdown.
            # handles the keystroke 2x CTRL+C to force an exit.
            self.stop_requested_force = True
            self.logger.warning('Force stopped: ' + str(sig))

            # just kill the process, we don't care about the results
            self.on_force_exit()
            os._exit(1)
            # with force_exit we really close the process, killing it in unknown state
            # self.fail('Force stopped', force_exit=True)
            # return

        if self.is_master_process():
            self.logger.warning('Received signal '+str(sig)+'. Send again to force stop. Stopping ...')
        else:
            self.logger.debug("Got child signal " + str(sig))

        self.stop_requested = True

        # the default SIGINT handle in python is not always installed, so we can't rely on the
        # KeyboardInterrupt exception to be thrown.
        # thread.interrupt_main would call sigint again.
        # the shutdown listener will do the rest like committing rest memory files into Git and closing connections.
        sys.exit(0 if self.in_early_stop else 1)

    def external_aborted(self, params):
        """
        Immediately abort the job by server.

        This runs in the Client:read() thread.
        """
        self.ended = True
        self.running = False

        # When the server sends an abort signal, we really have to close immediately,
        # since for example the job has been already deleted.
        # without touching the git and client any further
        os._exit(1)

    def external_stop(self, force):
        """
        Stop signal by server.
        """

        # only the master processes handles the regular stop signal from the server, sending a SIGINT to
        # all its child (means to us, non-master process)
        if not self.is_master_process():
            if force:
                # make sure even the subprocess dies really on force
                os._exit(1)

            return

        self.logger.warning("Received stop signal by server.")
        if not self.stop_requested_force:
            self.stop_requested_force = force

        raise_sigint()

    def early_stop(self):
        """
        Stop when a limitation is reached (like maxEpoch, maxTime).
        """
        self.in_early_stop = True

        raise_sigint()

    def batch(self, batch, total, size, label='BATCH', speed_label='SAMPLES/S'):
        self.step(batch, total, size=size, label=label, speed_label=speed_label)

    def sample(self, sample, total, label='SAMPLE', speed_label='SAMPLES/S'):
        self.step(sample, total, size=1, label=label, speed_label=speed_label)

    def step(self, step, total, label='STEP', speed_label='STEPS/S', size=1):
        """
        Increase the step indicator, which is a sub progress circle of the actual
        main progress circle (epoch, progress() method).
        """

        self.lock.acquire()
        try:
            time_diff = time.time() - self.last_step_time

            if self.last_step > step:
                # it restarted
                self.last_step = 0

            made_steps_since_last_call = step - self.last_step
            self.last_step = step

            self.made_steps_since_last_sync += made_steps_since_last_call
            self.made_steps_size_since_last_sync += made_steps_since_last_call * size

            if time_diff >= 1 or step == total:  # only each second or last batch
                self.set_system_info('step', step, True)
                self.set_system_info('steps', total, True)

                steps_per_second = self.made_steps_since_last_sync / time_diff
                samples_per_second = self.made_steps_size_since_last_sync / time_diff
                self.last_step_time = time.time()

                if size:
                    self.report_speed(samples_per_second)

                epochs_per_second = steps_per_second / total  # all batches
                self.set_system_info('epochsPerSecond', epochs_per_second, True)

                current_epochs = self.current_epoch if self.current_epoch else 1
                total_epochs = self.total_epochs if self.total_epochs else 1

                self.made_steps_since_last_sync = 0
                self.made_steps_size_since_last_sync = 0

                eta = 0
                if step < total:
                    # time to end this epoch
                    if steps_per_second != 0:
                        eta = (total - step) / steps_per_second

                # time until all epochs are done
                if total_epochs - current_epochs > 0:
                    if epochs_per_second != 0:
                        eta += (total_epochs - (current_epochs)) / epochs_per_second

                self.git.store_file('aetros/job/times/eta.json', simplejson.dumps(eta))

            if label and self.step_label != label:
                self.set_system_info('stepLabel', label, True)
                self.step_label = label

            if speed_label and self.step_speed_label != speed_label:
                self.set_system_info('stepSpeedLabel', speed_label, True)
                self.step_speed_label = speed_label
        finally:
            self.lock.release()

    def report_speed(self, speed, x=None, label=None):
        if not self.is_master_process():
            self.stdout_api_call('speed', x=x, speed=speed, label=label)
            return

        self.last_speed = speed
        self.last_speed_time = time.time()

        if x is None:
            x = round(time.time()-self.start_time, 3)

        self.set_system_info('samplesPerSecond', speed, True)
        self.speed_stream.write(simplejson.dumps([x, speed])[1:-1] + "\n")

        if label and self.step_speed_label != label:
            self.set_system_info('stepSpeedLabel', label, True)
            self.step_speed_label = label

    def stdout_api_call(self, command, **kwargs):
        action = {'aetros': command}
        action.update(kwargs)
        print(simplejson.dumps(action))

    @property
    def job_settings(self):
        if 'settings' in self.job['config']:
            return self.job['config']['settings']
        return {}

    def set_parameter_change_callback(self, callback):
        self.parameter_change_callback = callback

    def on_parameter_changed(self, params):
        pass

    def create_progress(self, name, total_steps=100):
        if name in self.progresses:
            return self.progresses[name]

        class Controller():
            def __init__(self, git, name, total_steps=100):
                self.started = False
                self.stopped = False
                self.lock = Lock()
                self.name = name
                self.step = 0
                self.steps = total_steps
                self.eta = 0
                self.last_call = 0
                self.git = git
                self._label = name

                self.store()

            def store(self):
                info = {
                    'label': self._label,
                    'started': self.started,
                    'stopped': self.stopped,
                    'step': self.step,
                    'steps': self.steps,
                    'eta': self.eta,
                }
                self.git.store_file('aetros/job/progress/' + self.name + '.json', simplejson.dumps(info))

            def label(self, label):
                self._label = label
                self.store()

            def start(self):
                if self.started is not False:
                    return

                self.step = 0
                self.started = time.time()
                self.last_call = time.time()
                self.store()

            def stop(self):
                if self.stopped is not False:
                    return

                self.stopped = time.time()
                self.store()

            def advance(self, steps=1):
                if steps <= 0:
                    return

                self.lock.acquire()
                if self.started is False:
                    self.start()

                took = (time.time() - self.last_call) / steps

                self.last_call = time.time()
                self.step += steps
                self.eta = took * (self.steps - self.step)

                if self.step >= self.steps:
                    self.stop()

                self.store()
                self.lock.release()

        self.progresses[name] = Controller(self.git, name, total_steps)

        return self.progresses[name]

    def epoch(self, epoch=None, total=None):
        self.progress(epoch, total)

    def progress(self, progress=None, total=None):
        self.current_epoch = self.current_epoch if progress is None else progress
        self.total_epochs = self.total_epochs if total is None else total
        epoch_limit = False

        if 'maxEpochs' in self.job['config'] and isinstance(self.job['config']['maxEpochs'], int) and self.job['config']['maxEpochs'] > 0:
            epoch_limit = True
            self.total_epochs = self.job['config']['maxEpochs']

        if self.current_epoch is not 0 and self.last_progress_call:
            # how long took it since the last call?
            time_per_epoch = time.time() - self.last_progress_call
            eta = time_per_epoch * (self.total_epochs - self.current_epoch)
            if self.current_epoch > self.total_epochs:
                eta = 0

            self.git.store_file('aetros/job/times/eta.json', simplejson.dumps(eta))

            if time_per_epoch > 0:
                self.set_system_info('epochsPerSecond', 1 / time_per_epoch, True)

        self.set_system_info('epoch', self.current_epoch, True)
        self.set_system_info('epochs', self.total_epochs, True)
        self.last_progress_call = time.time()

        if epoch_limit and self.total_epochs > 0:
            if self.current_epoch >= self.total_epochs:
                self.logger.warning("Max epoch of "+str(self.total_epochs)+" reached")
                self.early_stop()
                return

    def create_loss_channel(self, name='loss', xaxis=None, yaxis=None, layout=None):
        """
        :param name: string
        :return: JobLossGraph
        """

        return JobLossChannel(self, name, xaxis, yaxis, layout)

    def create_channel(self, name, traces=None,
                       main=False, kpi=False, kpiTrace=0, max_optimization=True,
                       type=JobChannel.NUMBER,
                       xaxis=None, yaxis=None, layout=None):
        """
        :param name: str
        :param traces: None|list : per default create a trace based on "name".
        :param main: bool : whether this channel is visible in the job list as column for better comparison.

        :param kpi: bool : whether this channel is the KPI (key performance indicator).
                           Used for hyperparameter optimization. Only one channel can be a kpi. Only first trace used.
        :param kpiTrace: bool : if you have multiple traces, define which is the KPI. 0 based index.

        :param max_optimization: bool : whether the optimization maximizes or minmizes the kpi. Use max_optimization=False to
                                        tell the optimization algorithm that his channel minimizes a kpi, for instance the loss of a model.

        :param type: str : One of JobChannel.NUMBER, JobChannel.TEXT
        :param xaxis: dict
        :param yaxis: dict
        :param layout: dict
        """
        return JobChannel(self, name, traces, main, kpi, kpiTrace, max_optimization, type, xaxis, yaxis, layout)

    def connect(self):
        self.client.configure(self.model_name, self.job_id, self.name)
        return self.client.start(['', 'files'])

    def start(self, collect_system=True, offline=False, push=True):
        if self.started:
            raise Exception('Job was already started.')

        if self.running:
            raise Exception('Job already running.')

        if not self.job_id:
            raise Exception('No job id found. Use create() first.')

        if not self.job:
            raise Exception('Job not loaded')

        prepend_signal_handler(signal.SIGINT, self.on_sigint)
        self.start_time = time.time()

        self.started = True
        self.running = True
        self.ended = False

        on_shutdown.started_jobs.append(self)

        self.client.configure(self.model_name, self.job_id, self.name)

        if not offline:
            # Marks client as active if not already. If not already starts to connect to the server
            self.client.start(['', 'files'])
        else:
            self.logger.debug('Job backend not started since offline.')

        if self.is_master_process():
            # this is the process that actually starts the job.
            # other sub-processes may only modify other data.
            self.git.commit_file('JOB_STARTED', 'aetros/job/times/started.json', simplejson.dumps(self.start_time))
            self.job_add_status('progress', JOB_STATUS.PROGRESS_STATUS_STARTED)
            self.git.store_file('aetros/job/times/elapsed.json', str(0))

            if collect_system:
                self.collect_system_information()
                self.collect_environment()

            # make sure we get the progress first, before monitoring sends elapses and
            # updates the job cache
            if not offline and self.client.is_online() and push:
                self.git.push()
                self.git.start_push_sync()

            if collect_system:
                self.start_monitoring()

            # log stdout to Git by using self.write_log -> git:stream_file
            self.stream_log = self.git.stream_file('aetros/job/log.txt', fast_lane=False)
            self.speed_stream = self.git.stream_file('aetros/job/speed.csv')
            header = ["x", "speed"]
            self.speed_stream.write(simplejson.dumps(header)[1:-1] + "\n")

            if isinstance(sys.stdout, GeneralLogger):
                sys.stdout.job_backend = self
                sys.stdout.flush()

            if isinstance(sys.stderr, GeneralLogger):
                sys.stderr.job_backend = self
                sys.stdout.flush()
        else:
            # if this process has been called within another process that is already using JobBackend.
            # we disable some stuff
            if isinstance(sys.stdout, GeneralLogger) and not sys.stderr.job_backend:
                sys.stdout.disable_buffer()

            if isinstance(sys.stderr, GeneralLogger) and not sys.stderr.job_backend:
                sys.stderr.disable_buffer()

    def set_paused(self, v):
        self.is_paused = v
        self.set_system_info('paused', self.is_paused, True)

    def is_master_process(self):
        """
        Master means that aetros.backend.start_job() has been called without using the command `aetros start`.
        If master is true, we collect and track some data that usually `aetros start` would do and reset the job's
        temp files on the server.
        :return:
        """

        return os.getenv('AETROS_JOB_ID') is None

    def detect_git_version(self, working_dir=None):
        current_dir = os.getcwd()

        try:
            if working_dir:
                os.chdir(working_dir)

            with self.git.batch_commit('Git Version'):
                value = git.get_current_remote_url()
                if value:
                    self.set_system_info('git_remote_url', value)

                value = git.get_current_commit_hash()
                if value:
                    self.set_system_info('git_version', value)

                value = git.get_current_branch()
                if value:
                    self.set_system_info('git_branch', value)

                value = git.get_current_commit_message()
                if value:
                    self.set_system_info('git_commit_message', value)

                value = git.get_current_commit_author()
                if value:
                    self.set_system_info('git_commit_author', value)
        finally:
            if working_dir:
                os.chdir(current_dir)

    def start_monitoring(self, cpu_cores=1, gpu_devices=None, docker_container=None):
        if not self.monitoring_thread:
            self.monitoring_thread = MonitoringThread(self, cpu_cores, gpu_devices, docker_container)
            self.monitoring_thread.daemon = True
            self.monitoring_thread.start()

    def create_keras_callback(self, model,
                              insights=False, insights_x=None,
                              additional_insights_layer=[],
                              confusion_matrix=False, validation_data=None, validation_data_size=None):

        """

        :type validation_data: int|None: (x, y) or generator
        :type validation_data_size: int|None: Defines the size of validation_data, if validation_data is a generator
        """

        if insights and (insights_x is None or insights_x is False):
            raise Exception('Can not build Keras callback with active insights but with invalid `insights_x` as input.')

        if confusion_matrix and (validation_data is None or validation_data is False):
            raise Exception('Can not build Keras callback with active confusion_matrix but with invalid `validation_data` as input.')

        from aetros.KerasCallback import KerasCallback
        self.callback = KerasCallback(self, self.logger, force_insights=insights)
        self.callback.insights_x = insights_x
        self.callback.insight_layer = additional_insights_layer
        self.callback.confusion_matrix = confusion_matrix
        self.callback.set_validation_data(validation_data, validation_data_size)

        return self.callback

    def upload_keras_graph(self, model):
        from aetros.keras import model_to_graph
        import keras

        if keras.__version__[0] == '2':
            graph = model_to_graph(model)
            self.set_graph(graph)

    def on_shutdown(self):
        """
        Shutdown routine. Sets the last progress (done, aborted, failed) and tries to send last logs and git commits.
        Also makes sure the ssh connection is closed (thus, the job marked as offline).

        Is triggered by atexit.register().
        """

        self.in_shutdown = True

        self.logger.debug('on_shutdown, stopped=%s, ended=%s, early_stop=%s, stop_requested=%s'
                          % (str(self.stopped), str(self.ended), str(self.in_early_stop), str(self.stop_requested)))
        if self.stopped or self.ended:
            # make really sure, ssh connection closed
            self.client.close()
            return

        if self.in_early_stop:
            self.done()
            return

        if self.stop_requested:
            # when SIGINT has been triggered
            if self.stop_requested_force:
                if not self.is_master_process():
                    # if not master process, we just stop everything. status/progress is set by master
                    self.stop(force_exit=True)
                else:
                    # master process
                    self.fail('Force stopped.', force_exit=True)
            else:
                if not self.is_master_process():
                    # if not master process, we just stop everything. status/progress is set by master
                    self.stop()
                else:
                    # master process
                    self.abort()

            return

        if hasattr(sys, 'last_value'):
            # sys.last_value contains a exception, when there was an uncaught one
            if isinstance(sys.last_value, KeyboardInterrupt):
                # can only happen when KeyboardInterrupt has been raised manually
                # since the one from the default sigint handler will never reach here
                # since we catch the sigint signal and sys.exit() before the default sigint handler
                # is able to raise KeyboardInterrupt
                self.abort()
            else:
                self.fail(type(sys.last_value).__name__ + ': ' + str(sys.last_value))

        elif self.running:
            self.done()

    def done(self, force_exit=False):
        if not self.running:
            return

        self.stop(JOB_STATUS.PROGRESS_STATUS_DONE, force_exit=force_exit)

    def send_std_buffer(self):
        if isinstance(sys.stdout, GeneralLogger):
            sys.stdout.send_buffer()

        if isinstance(sys.stderr, GeneralLogger):
            sys.stderr.send_buffer()

    def stop(self, progress=None, force_exit=False):
        global last_exit_code

        if self.stopped:
            return

        if self.is_master_process():
            self.section('Ended')

        self.logger.debug("stop: " + str(progress))

        self.send_std_buffer()

        self.stopped = True
        self.ended = True
        self.running = False

        if self.is_master_process() and progress is not None:
            # if not master process, the master process will set it
            self.job_add_status('progress', progress)

        exit_code = last_exit_code or 0
        if progress == JOB_STATUS.PROGRESS_STATUS_DONE:
            exit_code = 0

        if progress == JOB_STATUS.PROGRESS_STATUS_ABORTED:
            exit_code = 1

        if progress == JOB_STATUS.PROGRESS_STATUS_FAILED:
            exit_code = 2

        if self.is_master_process():
            self.set_system_info('exit_code', exit_code)

        # stop push thread and commit STREAMED/STORE END files in local git
        self.logger.debug("Git stopping ...")
        self.git.stop()

        if self.client.is_online() and not force_exit:
            # make sure all queues are empty and everything has been sent
            self.logger.debug("Wait for queue empty and store Git blobs on server: master=" +str(self.is_master_process()))

            # Say server to store received files as blob in remote Git
            self.client.send({'type': 'sync-blob'}, channel='')
            self.client.send({'type': 'sync-blob'}, channel='files')

            report = self.is_master_process() or is_debug()

            if report:
                sys.stdout.write("Uploading last job data ... ")

            self.client.wait_until_queue_empty(['', 'files'], report=report, clear_end=True)
            self.logger.debug("Blobs remotely stored, build latest git pack now")
            # all further client.send calls won't be included in the final git push calculation
            # and might be sent again.

            failure_in_last_sync = False

            # do now the last final git push, where we upload commits and trees.
            # blobs should be added already via streaming
            if self.is_master_process():
                # non-master commit and upload only.
                # master tracks what commits habe been sent already
                if self.git.push() is False:
                    failure_in_last_sync = True

            # send all last messages and git pack
            self.logger.debug("Last wait_until_queue_empty")
            self.client.wait_until_queue_empty(['', 'files'], report=report, clear_end=False)

            # it's important to have it here, since its tracks not only hardware but also network speed
            # for uploading last messages and Git.
            # Also, after each message we get from this thread on the server, we check if the job
            # should be ended/terminated or not.
            if self.monitoring_thread:
                self.monitoring_thread.stop()
                self.monitoring_thread.join()

            # send last monitoring stuff and close channels
            self.client.wait_sending_last_messages()

            if self.is_master_process():
                sys.stdout.write(" done.\n")

            # wait for end of client. Server will now close connection when ready.
            self.client.end()

            if self.is_master_process():
                # check if we have uncommitted stuff
                objects_to_sync, types = self.git.diff_objects(self.git.get_head_commit())

                if objects_to_sync:
                    failure_in_last_sync = True

            if self.is_master_process() and failure_in_last_sync:
                self.logger.warning("Not all job data have been uploaded.")
                self.logger.warning("Please run following command to make sure your job is stored on the server.")
                self.logger.warning("$ aetros job-push " + self.job_id[0:9])

        elif self.is_master_process():
            if force_exit:
                self.logger.warning("Not all job data have been uploaded because you force the exit.")
            else:
                self.logger.warning("Not all job data have been uploaded because you went offline.")

            self.logger.warning("Run following command to make sure your job is stored on the server.")
            self.logger.warning("$ aetros job-push " + self.job_id[0:9])

        if self.is_master_process():
            # remove the index file
            # non-master use the same as master, so master cleans up
            self.git.clean_up()

        # make sure client is really stopped
        self.client.close()

        self.logger.debug("Stopped %s with last commit %s." % (self.git.ref_head, self.git.get_head_commit()))

        if force_exit:
            self.on_force_exit()
            os._exit(exit_code)
        elif not self.in_shutdown:
            sys.exit(exit_code)

    def abort(self, force_exit=False):
        if not self.running:
            return

        self.set_status('ABORTED', add_section=False)

        self.stop(JOB_STATUS.PROGRESS_STATUS_ABORTED, force_exit=force_exit)

    def fail(self, message=None, force_exit=False):
        """
        Marks the job as failed, saves the given error message and force exists the process when force_exit=True.
        """
        global last_exit_code

        if not last_exit_code:
            last_exit_code = 1

        with self.git.batch_commit('FAILED'):
            self.set_status('FAILED', add_section=False)
            self.git.commit_json_file('FAIL_MESSAGE', 'aetros/job/crash/error', str(message) if message else '')
            if isinstance(sys.stderr, GeneralLogger):
                self.git.commit_json_file('FAIL_MESSAGE_LAST_LOG', 'aetros/job/crash/last_message', sys.stderr.last_messages)

        self.logger.debug('Crash report stored in commit ' + self.git.get_head_commit())
        self.stop(JOB_STATUS.PROGRESS_STATUS_FAILED, force_exit=force_exit)

    def write_log(self, message):
        """
        Proxy method for GeneralLogger.
        """
        if self.stream_log and not self.ended:
            # points to the Git stream write
            self.stream_log.write(message)
            return True

    def set_status(self, status, add_section=True):
        """
        Set an arbitrary status, visible in the big wheel of the job view.
        """
        status = str(status)

        if add_section:
            self.section(status)

        self.job_add_status('status', status)

    @property
    def job_id(self):
        return self.git.job_id

    @property
    def config(self):
        return self.job['config']

    def create(self, create_info=None, hyperparameter=None, server='local', insights=False):
        """
        Creates a new job in git and pushes it.

        :param create_info: from the api.create_job_info(id). Contains the config and job info (type, server)
        :param hyperparameter: simple nested dict with key->value, which overwrites stuff from aetros.yml
        :param server: if None, the the job will be assigned to a server.
        :param insights: whether you want to activate insights (for simple models)
        """
        if not create_info:
            create_info = {
                'server': server,
                'config': {
                    'insights': insights,
                    'command': ' '.join(sys.argv)
                }
            }
            config = find_config(self.config_path, logger=self.logger)

            if not config['model']:
                raise Exception('AETROS config file (aetros.yml) not found.')

            # first transform simple format in the full definition with parameter types
            # (string, number, group, choice_group, etc)
            full_hyperparameters = lose_parameters_to_full(config['parameters'])

            # now extract hyperparameters from full definition, and overwrite stuff using
            # incoming_hyperparameter if available
            hyperparameter = extract_parameters(full_hyperparameters, hyperparameter)
            create_info['config']['parameters'] = hyperparameter

        self.job = create_info

        if 'server' not in self.job and server:
            # setting this disables server assignment
            self.job['server'] = server

        self.job['optimization'] = None
        self.job['type'] = 'custom'

        if 'parameters' not in self.job['config']:
            self.job['config']['parameters'] = {}

        if 'insights' not in self.job['config']:
            self.job['config']['insights'] = insights

        self.job['created'] = time.time()
        self.git.create_job_id(self.job)

        self.logger.debug("Job created with Git ref " + self.git.ref_head)

        return self.job_id

    def is_simple_model(self):
        if not self.job:
            raise Exception('Job not loaded yet. Use load(id) first.')

        if 'type' in self.job:
            return self.job['type'] == 'simple'

        return False

    def ensure_model_name(self, model_name=None):
        if self.model_name:
            return

        if self.job and 'model' in self.job:
            return self.job['model']

        config = find_config(self.config_path, logger=self.logger)
        self.logger.debug('config: ' + simplejson.dumps(config))

        if model_name is None:
            model_name = os.getenv('AETROS_MODEL_NAME')

        if model_name is None:
            if 'model' not in config or not config['model']:
                sys.stderr.write('Error: No AETROS Trainer model name given. Specify it in aetros.yml `model: user/model-name` or use "aetros init model-name".\n')
                sys.exit(2)

            self.model_name = config['model']
        else:
            self.model_name = model_name

    def get_parameter(self, path, default=None, return_group=False):
        """
        Reads hyperparameter from job configuration. If nothing found use given default.

        :param path: str 
        :param default: *
        :param return_group: If true and path is a choice_group, we return the dict instead of the group name.
        :return: *
        """
        value = read_parameter_by_path(self.job['config']['parameters'], path, return_group)

        if value is None:
            return default

        return value

    def fetch(self, job_id):
        """
        Fetches the job from the server updating the job ref.
        """
        self.git.fetch_job(job_id)

    def load(self, job_id):
        """
        Loads job into index and work-tree, restart its ref and sets as current.

        :param job_id: int
        """
        self.git.read_job(job_id, checkout=self.is_master_process())
        self.load_job_from_ref()

    def load_job_from_ref(self):
        """
        Loads the job.json into self.job
        """
        if not self.job_id:
            raise Exception('Job not loaded yet. Use load(id) first.')

        if not os.path.exists(self.git.work_tree + '/aetros/job.json'):
            raise Exception('Could not load aetros/job.json from git repository. Make sure you have created the job correctly.')

        with open(self.git.work_tree + '/aetros/job.json') as f:
            self.job = simplejson.loads(f.read(), object_pairs_hook=collections.OrderedDict)

        if not self.job:
            raise Exception('Could not parse aetros/job.json from git repository. Make sure you have created the job correctly.')

        self.logger.debug('job: ' + str(self.job))

    def restart(self, job_id):
        self.git.read_job(job_id, checkout=True)

        progress = self.git.contents('aetros/job/status/progress.json')
        if progress is not None:
            progress = float(progress)
        else:
            progress = 0

        if progress >= 2:
            self.logger.error('You can not restart an existing job that was already running. You need to restart the '
                              'job through AETROS Trainer. progress='+str(progress))
            sys.exit(1)

        self.load_job_from_ref()

    def get_job_model(self):
        """
        Returns a new JobModel instance with current loaded job data attached.
        :return: JobModel
        """
        if not self.job:
            raise Exception('Job not loaded yet. Use load(id) first.')

        return JobModel(self.job_id, self.job, self.home_config['storage_dir'])

    def sync_weights(self, push=True):

        if not os.path.exists(self.get_job_model().get_weights_filepath_latest()):
            return

        self.logger.debug("sync weights...")
        self.set_status('SYNC WEIGHTS', add_section=False)

        with open(self.get_job_model().get_weights_filepath_latest(), 'rb') as f:
            import keras.backend
            self.git.commit_file('Added weights', 'aetros/weights/latest.hdf5', f.read())

            image_data_format = None
            if hasattr(keras.backend, 'set_image_data_format'):
                image_data_format = keras.backend.image_data_format()

            info = {
                'framework': 'keras',
                'backend': keras.backend.backend(),
                'image_data_format': image_data_format
            }
            self.git.commit_file('Added weights', 'aetros/weights/latest.json', simplejson.dumps(info))
            if push:
                self.git.push()

        # todo, implement optional saving of self.get_job_model().get_weights_filepath_best()

    def job_add_status(self, key, value):
        path = 'aetros/job/status/' + key + '.json'
        data = simplejson.dumps(value, default=invalid_json_values)
        self.git.commit_file('STATUS ' + str(value), path, data)

        if self.client.is_online():
            # just so have it faster
            self.client.send({'type': 'store-blob', 'path': path, 'data': data}, channel='')

    def set_info(self, name, value, commit_end_of_job=False):
        if commit_end_of_job:
            self.git.store_file('aetros/job/info/' + name + '.json',
                                simplejson.dumps(value, default=invalid_json_values))
        else:
            self.git.commit_json_file('INFO ' + name, 'aetros/job/info/' + name, value)

    def set_graph(self, graph):
        self.git.commit_json_file('GRAPH', 'aetros/job/graph', graph)

    def set_system_info(self, key, value, commit_end_of_job=False):
        if commit_end_of_job:
            self.git.store_file('aetros/job/system/' + key + '.json',
                                simplejson.dumps(value, default=invalid_json_values))
        else:
            self.git.commit_json_file('SYSTEM_INFO ' + key, 'aetros/job/system/' + key, value)

    def commit_file(self, path, git_path=None, title=None):
        path = os.path.expanduser(path).strip()

        if not git_path:
            git_path = os.path.relpath(path, os.getcwd())
            git_path = git_path.replace('../', '')
            git_path = git_path.replace('./', '')

        with self.git.batch_commit('FILE ' + (title or git_path)):
            if os.path.isdir(path):
                for file in os.listdir(path):
                    self.commit_file(path + '/' + file)
                return

            if os.path.getsize(path) > 10 * 1024 * 1024:
                self.logger.error('Can not upload files bigger than 10MB: ' + str(path))
                return

            with open(path, 'rb') as f:
                contents = f.read()

            self.git.commit_file('FILE ' + (title or git_path), git_path, contents)

    registered_actions = {}

    def register_action(self, callback, name=None, label=None, description=None):
        if name is None:
            name = callback.__name__

        args = {}
        inspect_args = inspect.getargspec(callback)
        if inspect_args.args:
            defaults = inspect_args.defaults if inspect_args.defaults else []

            start_default_idx = len(inspect_args.args) - len(defaults)

            for idx, argname in enumerate(inspect_args.args):
                args[argname] = {'default': None, 'type': 'mixed'}

                if idx >= start_default_idx:
                    default_value = defaults[idx - start_default_idx]
                    arg_type = 'mixed'
                    if isinstance(default_value, six.string_types): arg_type = 'string'
                    if isinstance(default_value, int): arg_type = 'integer'
                    if isinstance(default_value, float): arg_type = 'float'
                    if isinstance(default_value, bool): arg_type = 'bool'

                    args[argname] = {'default': default_value, 'type': arg_type}

        value = {
            'label': label,
            'description': description,
            'args': args,
            'instance': self.name
        }

        self.git.store_file('aetros/job/actions/' + name + '/config.json', simplejson.dumps(value, default=invalid_json_values))

        self.registered_actions[name] = value
        value['callback'] = callback

    def on_pause(self):
        pass

    def on_continue(self):
        pass

    def on_action(self, params):
        action_id = params['id']
        action_name = params['name']
        action_value = params['value']

        if action_name in ['pause', 'continue']:
            try:
                if action_name == 'pause': return self.on_pause()
                if action_name == 'continue': return self.on_continue()
            except SystemExit:
                raise
            except KeyboardInterrupt:
                raise
            except Exception as e:
                traceback.print_exc()
                self.logger.warning("Trigger action %s failed: %s" % (action_name, type(e).__name__ + ': ' + str(e)))

        if action_name not in self.registered_actions:
            # Received action ' + str(action_name) + ' but no callback registered.')
            return

        self.logger.debug("Trigger action: " + str(params))
        self.logger.info("Trigger action %s(%s)" %( action_name, str(action_value)))

        config = self.registered_actions[action_name]
        callback = config['callback']

        action = {
            'name': action_name,
            'value': action_value,
            'time': time.time(),
        }

        self.git.store_file(
            'aetros/job/actions/' + str(action_name) + '/result/' + str(action_id) + '.json',
            simplejson.dumps(action, default=invalid_json_values)
        )

        def done(value):
            result = {
                'value': value,
                'time': time.time()
            }
            # if value is binary, include mimetype and save it in a separate file

            self.git.store_file(
                'aetros/job/actions/' + str(action_name) + '/result/' + str(action_id) + '.json',
                simplejson.dumps(result, default=invalid_json_values)
            )

        kwargs = {}
        try:
            if action_value:
                kwargs = action_value

            if 'done' in config['args']:
                kwargs['done'] = done

            result = callback(**kwargs)
            # returning done as result marks this as async call

            if result is not done:
                # we have no async call
                done(result)

        except SystemExit:
            raise
        except KeyboardInterrupt:
            raise
        except Exception as e:
            traceback.print_exc()
            self.logger.warning("Trigger action %s(%s) failed: %s" % (action_name, str(kwargs), type(e).__name__+': '+ str(e)))

            result = {
                'exception': type(e).__name__,
                'message': str(e),
                'time': time.time()
            }

            self.git.store_file(
                'aetros/job/actions/' + str(action_name) + '/result/' + str(action_id) + '.json',
                simplejson.dumps(result, default=invalid_json_values)
            )

    def file_list(self):
        """
        Lists all files in the working directory.
        """
        blacklist = ['.git', 'aetros']
        working_tree = self.git.work_tree

        def recursive(path='.'):
            if os.path.basename(path) in blacklist:
                return 0, 0

            if os.path.isdir(path):
                files = []
                for file in os.listdir(path):
                    if path and path != '.':
                        file = path + '/' + file

                    added_files = recursive(file)
                    files += added_files

                return files
            else:
                if path.endswith('.pyc'):
                    return []

                if is_ignored(path, self.config['ignore']):
                    return []

                return [os.path.relpath(path, working_tree)]

        return recursive(working_tree)

    def add_files(self, working_tree, report=False):
        """
        Commits all files from limited in aetros.yml. `files` is a whitelist, `exclude_files` is a blacklist.
        If both are empty, we commit all files smaller than 10MB.
        :return:
        """
        blacklist = ['.git']

        def add_resursiv(path = '.', report=report):
            if os.path.basename(path) in blacklist:
                return 0, 0

            if working_tree + '/aetros' == path:
                # ignore in work_tree the folder ./aetros/, as it could be
                # that we checked out a job and start it again.
                return 0, 0

            if os.path.isdir(path):
                files = 0
                size = 0
                for file in os.listdir(path):
                    if path and path != '.':
                        file = path + '/' + file

                    added_files, added_size = add_resursiv(file)
                    files += added_files
                    size += added_size

                return files, size
            else:
                if path.endswith('.pyc'):
                    return 0, 0

                relative_path = os.path.relpath(path, working_tree)

                if is_ignored(relative_path, self.config['ignore']):
                    return 0, 0

                self.logger.debug("added file to job " + relative_path)
                if report:
                    print("Added job file: " + relative_path)

                self.git.add_file_path_in_work_tree(path, working_tree, verbose=False)

                return 1, os.path.getsize(path)

        return add_resursiv(working_tree, report=report)

    def add_embedding_word2vec(self, x, path, dimensions=None, header_with_dimensions=True):
        """
        Parse the word2vec file and extracts vectors as bytes and labels as TSV file.
        The format is simple: It's a UTF-8 encoded file, each word + vectors separated by new line.
        Vector is space separated.
        At the very first line might be dimensions, given as space separated value.


        Line 1: 2 4\n
        Line 2: word 200.3 4004.4 34.2 22.3\n
        Line 3: word2 20.0 4.4 4.2 0.022\n
        and so on

        For performance reasons, you should prefer add_embedding_path().

        """
        if path.endswith('.txt'):
            if not os.path.exists(path):
                raise Exception("Given word2vec file does not exist: " + path)

            f = open(path, 'r')

            if not header_with_dimensions and not dimensions:
                raise Exception('Either the word2vec file should contain the dimensions as header or it needs to be'
                                'specified manually using dimensions=[x,y] argument.')

            if header_with_dimensions:
                line = f.readline()
                if ' ' not in line:
                    raise Exception('Given word2vec file should have in first line the dimensions, e.g.: 1000 200')
                dimensions = np.fromstring(line, dtype=np.uint, sep=' ').tolist()

            labels = ''
            vectors = ''
            line_pos = 1 if header_with_dimensions else 0

            if len(dimensions) != 2:
                raise Exception('dimensions invalid shape. e.g. [200, 32] => 200 rows, 32 cols.')

            for line in iter(f.readline, ''):
                line_pos += 1
                space_pos = line.find(' ')
                if -1 == space_pos:
                    message = 'Given word2vec does not have correct format in line ' + str(line_pos)
                    message += '\nGot: ' + str(line)
                    raise Exception(message)

                labels += line[:space_pos] + '\n'
                vectors += line[space_pos+1:] + ' '

            vectors = np.fromstring(vectors, dtype=np.float32, sep=' ').tobytes()
        else:
            raise Exception("Given word2vec is not a .txt file. Other file formats are not supported.")

        info = {
            'dimensions': dimensions
        }

        name = os.path.basename(path)
        self._ensure_insight(x)
        remote_path = 'aetros/job/insight/'+str(x)+'/embedding/'
        with self.git.batch_commit('INSIGHT_EMBEDDING ' + str(x)):
            self.git.commit_file('WORD2VEC', remote_path + name + '/tensor.bytes', vectors)
            self.git.commit_file('WORD2VEC', remote_path + name + '/metadata.tsv', labels)
            self.git.commit_file('WORD2VEC', remote_path + name + '/info.json', simplejson.dumps(info))

    def add_embedding_path(self, x, dimensions, vectors_path, metadata=None, image_shape=None, image=None):
        """
        Adds a new embedding with optional metadata.
        
        Example how to generate vectors based on 2D numpy array:

        # 4 vectors, each size of 3
        vectors = [
            [2.3, 4.0, 33],
            [2.4, 4.2, 44],
            [2.5, 3.9, 34],
            [5.5, 200.2, 66]
        ]

        metadata = [
            # header, only necessary when more then on column
            # can be anything.
            ['label', 'count'],

            # for each vector from above an entry.
            ['red', '123'],
            ['white', '143'],
            ['yellow', '344'],
            ['house', '24'],
        ]

        numpy.array(vectors, dtype=numpy.float32).tofile('vectors.bytes')
        numpy.savetxt('metadata.tsv', numpy.array(metadata), delimiter='\t', fmt='%s')

        job.add_embedding_path([4, 3], 'vectors.bytes', 'metadata.tsv')

        Metadata format example:

        Label\tCount\n
        red\t4\n
        yellow\t6\n

        :param x: The x axis of the insights. 
        :param dimensions: 2D List of dimension, e.g [200, 20], means 200 vectors and each vector has size of 20.
        
        :param vectors_path: A path to a floats64 bytes file, no separators, sum(dimensions)*floats64 long.
                       Example: If dimensions [200, 20] then the tensor file has 200*20 float32 bytes in it
                        
        :param metadata: A TSV file. If only one column long (=no tab separator per line), then there's no need for a header.
                         If you have more than one column, use the first line as header.
                       
        :param image_shape: Size of the image of each vector.
        :param image: Path to an image sprite.
        :return: 
        """
        if not os.path.exists(vectors_path):
            raise Exception("Given embedding vectors file does not exist: " + vectors_path)

        if metadata and not os.path.exists(metadata):
            raise Exception("Given embedding metadata file does not exist: " + metadata)

        name = os.path.basename(vectors_path)
        self._ensure_insight(x)
        remote_path = 'aetros/job/insight/'+str(x)+'/embedding/'

        info = {
            'dimensions': dimensions,
            'image_shape': image_shape,
            'image': os.path.basename(image) if image else None,
        }

        with self.git.lock_write():
            self.git.add_file_path(remote_path + name + '/tensor.bytes', vectors_path)
            self.git.add_file_path(remote_path + name + '/metadata.tsv', metadata)
            self.git.add_file(remote_path + name + '/info.json', simplejson.dumps(info))

            if image:
                self.git.add_file(remote_path + name + '/' + os.path.basename(image), image)

            self.git.commit_index('INSIGHT_EMBEDDING ' + str(x))

    def add_insight_image_path(self, x, path, name=None, label=None):
        image = PIL.Image.open(path)

        if not name:
            name = os.path.basename(path)

        return self.add_insight_image(x, JobImage(name, image, label))

    def add_insight_image(self, x, image):
        self.add_insight_images(x, [image])

    def add_insight_images(self, x, images):
        converted_images = []

        self._ensure_insight(x)

        if x not in self.insight_images_info:
            self.insight_images_info[x] = {}

        for image in images:
            if not isinstance(image, JobImage):
                raise Exception('job_add_insight only accepts JobImage instances in images argument')

            if image.id in self.insight_images_info[x]:
                continue

            converted_images.append({
                'id': image.id,
                'image': self.pil_image_to_jpeg(image.image)
            })

            self.insight_images_info[x][image.id] = {
                'file': image.id+'.jpg',
                'label': image.label,
                'pos': image.pos
            }

        with self.git.batch_commit('INSIGHT_IMAGES ' + str(x)):
            for image in converted_images:
                remote_path = 'aetros/job/insight/'+str(x)+'/image/'+image['id']+'.jpg'
                self.git.commit_file('IMAGE ' + str(image['id']), remote_path, image['image'])

            remote_path = 'aetros/job/insight/' + str(x) + '/info.json'
            self.git.commit_file('IMAGE INFO', remote_path, simplejson.dumps(self.insight_images_info[x]))

    def add_insight_confusion_matrix(self, x, confusion_matrix):
        self._ensure_insight(x)
        remote_path = 'aetros/job/insight/' + str(x) + '/confusion_matrix.json'
        self.git.commit_file('INSIGHT CONFUSION_MATRIX ' + str(x), remote_path, simplejson.dumps(confusion_matrix))

    def job_add_insight(self, x, images=None, confusion_matrix=None):
        if images:
            self.add_insight_images(x, images)

        if confusion_matrix:
            self.add_insight_confusion_matrix(x, confusion_matrix)

    def _ensure_insight(self, x):
        if x in self.insight_created: return

        self.insight_created.append(x)
        remote_path = 'aetros/job/insight/' + str(x) + '/created'
        self.git.commit_file('WORD2VEC ' + str(x), remote_path, str(time.time()))

    def pil_image_to_jpeg(self, image):
        buffer = six.BytesIO()

        image.save(buffer, format="JPEG", optimize=True, quality=70)
        return buffer.getvalue()

    def collect_environment(self, overwrite_variables=None):
        import socket
        import os
        import pip
        import platform

        env = {}

        if not overwrite_variables:
            overwrite_variables = {}

        import aetros
        env['aetros_version'] = aetros.__version__
        env['python_version'] = platform.python_version()
        env['python_executable'] = sys.executable

        env['hostname'] = socket.gethostname()
        env['variables'] = dict(os.environ)
        env['variables'].update(overwrite_variables)

        if 'AETROS_SSH_KEY' in env['variables']: del env['variables']['AETROS_SSH_KEY']
        if 'AETROS_SSH_KEY_BASE64' in env['variables']: del env['variables']['AETROS_SSH_KEY_BASE64']

        env['pip_packages'] = sorted([[i.key, i.version] for i in pip.get_installed_distributions()])
        self.set_system_info('environment', env)

    def collect_device_information(self, gpu_ids):
        import aetros.cuda_gpu

        try:
            if gpu_ids:
                self.set_system_info('cuda_version', aetros.cuda_gpu.get_version())
                gpus = {}
                for gpu_id, gpu in enumerate(aetros.cuda_gpu.get_ordered_devices()):
                    if gpu_id in gpu_ids:
                        gpus[gpu_id] = gpu

                self.set_system_info('gpus', gpus)
        except CudaNotImplementedException:
            self.logger.warning("Could not collect GPU/CUDA system information.")

        if self.get_job_model().has_dpu():
            self.set_system_info('dpus', [{'memory': 64*1024*1024*1024}])

    def collect_system_information(self):
        import psutil

        mem = psutil.virtual_memory()

        with self.git.batch_commit('JOB_SYSTEM_INFORMATION'):
            self.set_system_info('memory_total', mem.total)

            import cpuinfo
            cpu = cpuinfo.get_cpu_info()
            self.set_system_info('cpu_name', cpu['brand'])
            self.set_system_info('cpu', [cpu['hz_actual_raw'][0], cpu['count']])

    stdout_api_channels = {}

    def handle_stdout_api(self, data):
        action = data['aetros']
        del data['aetros']

        def validate_action(requires_attributes):
            for attr in requires_attributes:
                if attr not in data:
                    raise StdoutApiException("AETROS stdout API call %s requires value for '%s'. " % (action, attr))

            return True

        def failed(message):
            raise StdoutApiException(
                "AETROS stdout API call %s failed: %s Following ignored: %s" % (action, message, str(data)))

        def default(attr, default=None):
            return data[attr] if attr in data else default

        if action == 'progress':
            self.progress(**data)
            return True

        if action == 'epoch':
            self.epoch(**data)
            return True

        if action == 'batch':
            if validate_action(['batch', 'total', 'size']):
                self.batch(**data)
                return True

        if action == 'step':
            if validate_action(['step', 'total']):
                self.step(**data)
                return True

        if action == 'sample':
            if validate_action(['sample', 'total']):
                self.sample(**data)
                return True

        if action == 'info':
            if validate_action(['name', 'value']):
                self.set_info(**data)
                return True

        if action == 'status':
            if validate_action(['status']):
                self.set_status(**data)
                return True

        if action == 'speed':
            if validate_action(['x', 'speed']):
                self.report_speed(**data)
                return True

        if action == 'add_embedding_word2vec':
            if validate_action(['x', 'path']):
                self.add_embedding_word2vec(**data)
                return True

        if action == 'add_embedding_path':
            if validate_action(['x', 'dimensions', 'vectors_path']):
                self.add_embedding_path(**data)
                return True

        if action == 'add_insight_image':
            if validate_action(['x', 'path']):
                self.add_insight_image_path(**data)
                return True

        if action == 'create-channel':
            if validate_action(['name']):
                if data['name'] in self.stdout_api_channels:
                    failed("Channel %s already defined. " % (data['name'], ))
                else:
                    self.stdout_api_channels[data['name']] = self.create_channel(**data)
                    return True

        if action == 'channel':
            if validate_action(['name', 'x', 'y']):
                if data['name'] not in self.stdout_api_channels:
                    self.stdout_api_channels[data['name']] = self.create_channel(data['name'])

                self.stdout_api_channels[data['name']].send(data['x'], data['y'])
                return True

        if action == 'loss':
            if validate_action(['x', 'training', 'validation']):
                if 'loss' not in self.stdout_api_channels:
                    self.stdout_api_channels['loss'] = self.create_loss_channel('loss')

                self.stdout_api_channels['loss'].send(data['x'], data['training'], data['validation'])
                return True

        # if action == 'insight':
        #     if validate_action(['x']):
        #
        #         self.job_add_insight(data['x'])
        #         return True

        if action == 'abort':
            self.abort()

        if action == 'fail':
            self.fail(default('message'))

        return False
