"""scikit-learn base"""

# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/06_sk.base.ipynb.

# %% auto 0
__all__ = ['DEFAULT_TOKEN_PATTERN', 'NOSPACE_LANGS', 'detect_encoding', 'decode_by_line', 'detect_lang', 'is_nospace_lang',
           'is_chinese', 'split_chinese', 'get_random_colors']

# %% ../../nbs/06_sk.base.ipynb 3
"""
sk base
"""

import numpy as np
import pandas as pd
import warnings

import charset_normalizer as chardet
import langdetect

DEFAULT_TOKEN_PATTERN = (
    r"\b[a-zA-Z][a-zA-Z0-9]*(?:[_/&-][a-zA-Z0-9]+)+\b|"
    r"\b\d*[a-zA-Z][a-zA-Z0-9][a-zA-Z0-9]+\b"
)

NOSPACE_LANGS = ["zh-cn", "zh-tw", "ja"]

def detect_encoding(texts, sample_size=32):
    if not isinstance(texts, list):
        # check for instance of list as bytes are supplied as input
        texts = [texts]
    lst = [chardet.detect(doc)["encoding"] for doc in texts[:sample_size]]
    encoding = max(set(lst), key=lst.count)
    # standardize to utf-8 to prevent BERT problems
    encoding = "utf-8" if encoding.lower() in ["ascii", "utf8", "utf-8"] else encoding
    return encoding

def decode_by_line(texts, encoding="utf-8", verbose=1):
    """
    ```
    Decode text line by line and skip over errors.
    ```
    """

    if isinstance(texts, str):
        texts = [texts]
    new_texts = []
    skips = 0
    num_lines = 0
    for doc in texts:
        text = ""
        for line in doc.splitlines():
            num_lines += 1
            try:
                line = line.decode(encoding)
            except:
                skips += 1
                continue
            text += line
        new_texts.append(text)
    pct = round((skips * 1.0 / num_lines) * 100, 1)
    if verbose:
        print("skipped %s lines (%s%%) due to character decoding errors" % (skips, pct))
        if pct > 10:
            print("If this is too many, try a different encoding")
    return new_texts


def detect_lang(texts:list, sample_size:int=32):
    """
    detect language of texts
    """

    # convert sentence pairs
    if isinstance(texts, (tuple, list, np.ndarray)) and len(texts) == 2:
        texts = [texts[0], texts[1]]
    elif (
        isinstance(texts, (tuple, list, np.ndarray))
        and isinstance(texts[0], (tuple, list, np.ndarray))
        and len(texts[0]) == 2
    ):
        texts = [t[0] for t in texts]

    if isinstance(texts, (pd.Series, pd.DataFrame)):
        texts = texts.values
    if isinstance(texts, str):
        texts = [texts]
    if not isinstance(texts, (list, np.ndarray)):
        raise ValueError("texts must be a list or NumPy array of strings")
    lst = []
    for doc in texts[:sample_size]:
        try:
            lst.append(langdetect.detect(doc))
        except:
            continue
    if len(lst) == 0:
        warnings.warn(
            "Defaulting to English for language detection: could not detect language from documents. "
            + "This may be due to empty or invalid texts being provided to detect_lang."
        )
        lang = "en"
    else:
        lang = max(set(lst), key=lst.count)
    return lang


def is_nospace_lang(lang):
    return lang in NOSPACE_LANGS

def is_chinese(lang, strict=True):
    """
    ```
    Args:
      lang(str): language code (e.g., en)
      strict(bool):  If False, include additional languages due to mistakes on short texts by langdetect
    ```
    """
    if strict:
        extra_clause = False
    else:
        extra_clause = lang in ["ja", "ko"]
    return lang is not None and lang.startswith("zh-") or extra_clause

def split_chinese(texts):
    try:
        import jieba
    except ImportError:
        raise ImportError('Please install jieba: pip install jieba')
        
    if isinstance(texts, str):
        texts = [texts]

    split_texts = []
    for doc in texts:
        seg_list = jieba.cut(doc, cut_all=False)
        seg_list = list(seg_list)
        split_texts.append(seg_list)
    return [" ".join(tokens) for tokens in split_texts]

def get_random_colors(n, name="hsv", hex_format=True):
    """Returns a function that maps each index in 0, 1, ..., n-1 to a distinct
    RGB color; the keyword argument name must be a standard mpl colormap name."""
    from matplotlib import pyplot as plt
    cmap = plt.cm.get_cmap(name, n)
    result = []
    for i in range(n):
        color = cmap(i)
        if hex_format:
            color = rgb2hex(color)
        result.append(color)
    return np.array(result)

