text:
"""
Profiling tools for use with PyXLL.

This code accompanies the blog post
https://www.pyxll.com/blog/how-to-profile-python-code-in-excel
"""
from pyxll import xl_menu, xl_app, xlcAlert
from win32com.client import constants
from functools import wraps
from ctypes import c_int64, byref, windll
import win32clipboard
import line_profiler
import cProfile
import pstats
import math

try:
    from io import StringIO
except ImportError:
    from StringIO import StringIO

_active_cprofiler = None
_active_line_profiler = None

def _median(values):
    """Return the median of a list of values"""
    n = len(values)
    if n < 1:
        return 0.0
    if n % 2 == 1:
        return sorted(values)[n//2]
    return sum(sorted(values)[n//2-1:n//2+1])/2.0

@xl_menu("Time to Calculate", menu="Profiling Tools")
def time_calculation():
    """Recalculates the selected range and times how long it takes"""
    xl = xl_app()

    orig_calc_mode = xl.Calculation
    try:
        # switch Excel to manual calculation and disable screen updating
        xl.Calculation = constants.xlManual
        xl.ScreenUpdating = False

        # int64 variables used for timing
        start_time = c_int64()
        end_time = c_int64()

        # Get the current selection and its Calculate method (to avoid including the
        # method retrieval in the timing)
        selection = xl.Selection
        selection_Calculate = selection.Calculate

        # run the calculation a few times
        timings = []
        for i in range(100):
            # Time calling selection.Calculate() using the Windows high-resolution timers
            windll.Kernel32.QueryPerformanceCounter(byref(start_time))
            selection_Calculate()
            windll.Kernel32.QueryPerformanceCounter(byref(end_time))
            duration = float(end_time.value - start_time.value)
            timings.append(duration)
    finally:
        # restore the original calculation mode and screen updating
        xl.ScreenUpdating = True
        xl.Calculation = orig_calc_mode

    # calculate the mean and stddev
    mean = math.fsum(timings) / len(timings)
    median = _median(timings)
    stddev = (math.fsum([(x - mean) ** 2 for x in timings]) / len(timings)) ** 0.5
    best = min(timings)
    worst = max(timings)

    # convert to seconds
    freq = c_int64()
    windll.Kernel32.QueryPerformanceFrequency(byref(freq))
    mean /= freq.value
    median /= freq.value
    stddev /= freq.value
    best /= freq.value
    worst /= freq.value

    # copy the results to the clipboard
    data = [
        ["mean", mean],
        ["median", median],
        ["stddev", stddev],
        ["best", best],
        ["worst", worst]
    ]
    text = "\n".join(["\t".join(map(str, x)) for x in data])
    win32clipboard.OpenClipboard()
    win32clipboard.EmptyClipboard()
    win32clipboard.SetClipboardText(text)
    win32clipboard.CloseClipboard()

    # report the results
    xlcAlert(("%0.2f ms \xb1 %d \xb5s\n"
              "Median: %0.2f ms\n"
              "Best: %0.2f ms\n"
              "Worst: %0.2f ms\n"
              "(Copied to clipboard)") % (mean * 1000, stddev * 1000000, median * 1000, best * 1000, worst * 1000))

@xl_menu("Start", menu="Profiling Tools", sub_menu="cProfile")
def start_profiling():
    """Start the cProfile profiler"""
    global _active_cprofiler
    if _active_cprofiler is not None:
        _active_cprofiler.disable()
    _active_cprofiler = cProfile.Profile()

    xlcAlert("cProfiler Active\n"
             "Recalcuate the workbook and then stop the profiler\n"
             "to see the results.")

    _active_cprofiler.enable()

@xl_menu("Stop", menu="Profiling Tools", sub_menu="cProfile")
def stop_profiling():
    """Stop the cProfile profiler and print the results"""
    global _active_cprofiler
    if not _active_cprofiler:
        xlcAlert("No active profiler")
        return

    _active_cprofiler.disable()

    # print the profiler stats
    stream = StringIO()
    stats = pstats.Stats(_active_cprofiler, stream=stream).sort_stats("cumulative")
    stats.print_stats()

    # print the results to the log
    print(stream.getvalue())

    # and copy to the clipboard
    win32clipboard.OpenClipboard()
    win32clipboard.EmptyClipboard()
    win32clipboard.SetClipboardText(stream.getvalue())
    win32clipboard.CloseClipboard()

    _active_cprofiler = None

    xlcAlert("cProfiler Stopped\n"
             "Results have been written to the log and clipboard.")

# Current active line profiler
_active_line_profiler = None

def enable_line_profiler(func):
    """Decorator to switch on line profiling for a function.

    If using line_profiler from the command line, use the built-in @profile
    decorator instead of this one.
    """
    @wraps(func)
    def wrapper(*args, **kwargs):
        nonlocal func
        if _active_line_profiler:
            func = _active_line_profiler(func)
        return func(*args, **kwargs)
    return wrapper

@xl_menu("Start", menu="Profiling Tools", sub_menu="Line Profiler")
def start_line_profiler():
    """Start the line profiler"""
    global _active_line_profiler
    _active_line_profiler = line_profiler.LineProfiler()

    xlcAlert("Line Profiler Active\n"
             "Run the function you are interested in and then stop the profiler.\n"
             "Ensure you have decoratored the function with @enable_line_profiler.")

@xl_menu("Stop", menu="Profiling Tools", sub_menu="Line Profiler")
def stop_line_profiler():
    """Stops the line profiler and prints the results"""
    global _active_line_profiler
    if not _active_line_profiler:
        return

    stream = StringIO()
    _active_line_profiler.print_stats(stream=stream)
    _active_line_profiler = None

    # print the results to the log
    print(stream.getvalue())

    # and copy to the clipboard
    win32clipboard.OpenClipboard()
    win32clipboard.EmptyClipboard()
    win32clipboard.SetClipboardText(stream.getvalue())
    win32clipboard.CloseClipboard()

    xlcAlert("Line Profiler Stopped\n"
             "Results have been written to the log and clipboard.")


text:
; To try this out, add this file to your existing pyxll.cfg
; file using the 'external_config' option.
;
;This code accompanies the blog post
;https://www.pyxll.com/blog/how-to-profile-python-code-in-excel

[PYTHON]
pythonpath = .

[PYXLL]
modules =
    profiling_tools
    average_tol

repo_name: pyxll/pyxll-examples

repo_language: Python

mime_type: text/plain


text:
"""
Excel functions "publish" and "subscribe" for passing
values between workbooks.

An RTD function is used for subscribe so that published
values are updated in read time.
"""
from pyxll import xl_func, RTD
import pubsub

class SubscriberRTD(RTD):
    """RTD class that updates whenever a value is published
    on a given topic."""

    def __init__(self, topic):
        super().__init__(value="Waiting...")
        self.__topic = topic

    def connect(self):
        pubsub.subscribe(self.__topic, self.__callback)

    def disconnect(self):
        pubsub.unsubscribe(self.__topic, self.__callback)

    def __callback(self, value):
        self.value = value

@xl_func
def publish(topic, value):
    """Publish a value on a topic to be picked up by another sheet"""
    pubsub.publish(topic, value)
    return f"[Published to {topic}]"

@xl_func("str: rtd")
def subscribe(topic):
    """Get a value that has been published on another sheet"""
    return SubscriberRTD(topic)

#
# Example application:
#
# One sheet constructs a swap curve object and publishes it.
# Other sheets can subscribe to the topic the curve object
# is published on to get the curve, without having to
# reference the sheet directly or use INDEX.
#

class DummySwapCurve:
    def __init__(self, name):
        self.__name = name

@xl_func(recalc_on_open=True)
def create_swap_curve(name, args):
    """Creates a new swap curve and returns it to Excel"""
    return DummySwapCurve(name)


text:
"""
A simple in-process message broker to decouple
components in a single Python application.
"""
import threading
import logging

_log = logging.getLogger(__name__)

class MessageBroker:
    """The message broker waits for topics to be published and then
    notifies any subscribers of the published topics.

    When a new subscriber subscribes they will be immediately sent the latest
    value if there is one.

    Only the latest value is used and so publishing multiple messages
    on the same topic may result in the subscriber only seeing the
    latest value. This is the intended behaviour.
    """

    def __init__(self):
        # Dict of current values for each topic.
        self.__latest_values = {}

        # Lists of callbacks for each topic subscribed to
        self.__subscribers = {}

        # Prevent concurrent access to the above data structures
        self.__lock = threading.RLock()

    def subscribe(self, topic, callback):
        """Subscribes to a topic.
        The subscriber callback is called with the topic value whenever
        the a value is published to that topic, and with the initial
        value if there is one.
        """
        with self.__lock:
            # Add the subscriber to the list of subscribers
            subscribers = self.__subscribers.setdefault(topic, [])
            subscribers.append(callback)

            # Get the current value for the topic, if there is one and call the callback
            value = self.__latest_values.get(topic)
            if value is not None:
                callback(value)

    def unsubscribe(self, topic, callback):
        """Unsubscribe from a topic.
        The topic and callback must have previously be passed to 'subscribe'.
        """
        with self.__lock:
            # Remove the callback from the list of subscribers
            subscribers = self.__subscribers[topic]
            subscribers.remove(callback)

            if not subscribers:
                del self.__subscribers[topic]

    def publish(self, topic, value):
        """Publish a value to all subscribers of a topic.
        """
        with self.__lock:
            # Call the subscribers for the published topic
            if value is not None:
                for callback in self.__subscribers.get(topic, []):
                    try:
                        callback(value)
                    except:
                        _log.error(f"Failed to notify subscriber of {topic}", exc_info=True)

            # Update the latest value for any new subscribers
            self.__latest_values[topic] = value

# Use a single global message broker for all topics
_global_message_broker = MessageBroker()

def publish(topic, value):
    """Publishes a value on a topic for a subscriber to receive"""
    _global_message_broker.publish(topic, value)

def subscribe(topic, callback):
    """Subscribes to a topic.
    The subscriber callback is called with the topic value whenever
    the a value is published to that topic, and with the initial
    value if there is one.
    """
    _global_message_broker.subscribe(topic, callback)

def unsubscribe(topic, callback):
    """Unsubscribe from a topic.
    The topic and callback must have previously be passed to 'subscribe'.
    """
    _global_message_broker.unsubscribe(topic, callback)


text:
; To try this out, add this file to your existing pyxll.cfg
; file using the 'external_config' option.
;
; This code accompanies the Medium post
; https://medium.com/@tony_86605/a-non-volatile-indirect-alternative-in-excel-using-the-pub-sub-pattern-15cea21272a3

[PYXLL]
modules =
    pubsub_example

[PYTHON]
pythonpath =
    ./


text:
# A Non-Volatile INDIRECT Alternative in Excel using the Pub/Sub Pattern

This code accompanies the Medium post
[A Non-Volatile INDIRECT Alternative in Excel using the Pub/Sub Pattern](https://medium.com/@tony_86605/a-non-volatile-indirect-alternative-in-excel-using-the-pub-sub-pattern-15cea21272a3)

Messages are posted to a message broker from one sheet and subscribed to in another
to pass values between Excel workbooks.

Updates to values are published and updated in the subscribing sheets(s) in
real time using an RTD function.

Unlike the Excel INDEX function this does not use a volatile function and
so doesn't suffer from the same potential performance problems that sheets
using INDEX do.


text:
"""
Example code to show how to use the 'Running Object Table' to find the right
Excel Application object to use from a child process.

When spawning a child process (or processes) from Excel sometimes it's
useful to be able to get a reference to the Excel Applciation object
corresponding to the parent process (to write back to a worksheet for
example).

The Excel application object can be obtained by using
win32com.client.Dispatch("Excel.Application") but this isn't guarenteed
to be the correct instance of Excel if there are multiple Excel applications
running.

This example shows how to solve this problem using the 'Running Object Table'.

References:
https://msdn.microsoft.com/en-us/library/windows/desktop/ms695276%28v=vs.85%29.aspx
https://msdn.microsoft.com/en-us/library/windows/desktop/ms684004(v=vs.85).aspx
"""
from pyxll import xl_func, get_active_object
from multiprocessing import Process
import win32com.client
import pythoncom
import logging
import time
import os

IID_Workbook = pythoncom.MakeIID("{000208DA-0000-0000-C000-000000000046}")

def get_xl_app(parent=None):
    """
    Return an Excel Application instance.

    Unlike using win32com.client.Dispatch("Excel.Application") the
    Application returned will always be the one that corresponds
    to the parent process.
    """
    # Get the window handle set by the parent process
    parent_hwnd = os.environ["PYXLL_EXCEL_HWND"]

    # Iterate over the running object table looking for the Excel Workbook
    # object from the parent process' Application object.
    context = pythoncom.CreateBindCtx(0)
    for moniker in pythoncom.GetRunningObjectTable():
        try:
            # Workbook implements IOleWindow so only consider objects implementing that
            window = moniker.BindToObject(context, None, pythoncom.IID_IOleWindow)
            disp = window.QueryInterface(pythoncom.IID_IDispatch)

            # Get a win32com Dispatch object from the PyIDispatch object as it's
            # easier to work with.
            obj = win32com.client.Dispatch(disp)

        except pythoncom.com_error:
            # Skip any objects we're not interested in
            continue

        # Check the object we've found is a Workbook
        if getattr(obj, "CLSID", None) == IID_Workbook:
            # Get the Application from the Workbook and if its window matches return it.
            xl_app = obj.Application
            if str(xl_app.Hwnd) == parent_hwnd:
                return xl_app

    # This can happen if the parent process has terminated without terminating
    # the child process.
    raise RuntimeError("Parent Excel application not found")

def _subprocess_func(target_address, logfile):
    """
    This function is run in a child process of the main Excel process.
    """
    # Initialize logging (since we're now running outside of Excel this isn't done for us).
    logging.basicConfig(filename=logfile, level=logging.INFO)
    log = logging.getLogger(__name__)
    log.info("Child process %d starting" % os.getpid())

    try:
        # Get the Excel Application corresponding to the parent process
        xl_app = get_xl_app()

        # Write to the target cell in the parent Excel.
        cell = xl_app.Range(target_address)
        message = "Child process %d is running..." % os.getpid()
        cell.Value = message

        # Run for a few seconds updating the value periodically
        for i in range(300):
            message = message[1:] + message[0]

            # When setting a value in Excel it may fail if the user is also
            # interacting with the sheet.
            try:
                cell.Value = message
            except:
                log.warn("Error setting cell value", exc_info=True)

            time.sleep(0.2)

        cell.Value = "Child process %d has terminated" % os.getpid()

    except Exception:
        log.error("An error occured in the child process", exc_info=True)
        raise

@xl_func("string target_address: string")
def start_subprocess(target_address):
    """
    Start a sub-process that will write back to a cell.
    :param target_address: address of cell to write to from the child process.
    """
    # Get the window handle of the Excel process so the sub-process can
    # find the right Excel Application instance.
    xl_app = win32com.client.Dispatch(get_active_object()).Application
    os.environ["PYXLL_EXCEL_HWND"] = str(xl_app.Hwnd)

    # Get the log file name for the subproces to log to.
    root = logging.getLogger()
    logfile = None
    for handler in root.handlers:
        if isinstance(handler, logging.FileHandler):
            logfile = handler.baseFilename
            break

    # Start the subprocess that will write back to the target cell.
    # It's a daemon process so that it doesn't stop the main Excel process
    # from terminating even if it's still running.
    process = Process(target=_subprocess_func, args=(target_address, logfile))
    process.daemon = True
    process.start()

    return "Child process %d started" % process.pid


text:
"""
Functions for display charts in Excel using Highcharts.

Please be aware that the Highcharts project itself, as well as Highmaps and Highstock,
are only free for non-commercial use under the Creative Commons Attribution-NonCommercial license.
Commercial use requires the purchase of a separate license. Pop over to Highcharts for more information.

This code accompanies the blog post
https://www.pyxll.com/blog/interactive-charts-in-excel-with-highcharts
"""
from pyxll import xl_func, xl_app, xlfCaller
from highcharts.highstock.highstock_helper import jsonp_loader
from bs4 import BeautifulSoup
import tempfile
import timer
import re
import os

def hc_plot(chart, control_name, theme=None):
    """
    This function is used by the other plotting functions to render the chart as html
    and display it in Excel.
    """
    # add the theme if there is one
    if theme:
        chart.add_JSsource(["https://code.highcharts.com/themes/%s.js" % theme])

    # get the calling sheet
    caller = xlfCaller()
    sheet_name = caller.sheet_name

    # split into workbook and sheet name
    match = re.match("^\[(.+?)\](.*)$", sheet_name.strip("'\""))
    if not match:
        raise Exception("Unexpected sheet name '%s'" % sheet_name)
    workbook, sheet = match.groups()

    # get the Worksheet object
    xl = xl_app()
    workbook = xl.Workbooks(workbook)
    sheet = workbook.Sheets(sheet)

    # find the existing webbrowser control, or create a new one
    try:
        control = sheet.OLEObjects(control_name[:31])
        browser = control.Object
    except:
        control = sheet.OLEObjects().Add(ClassType="Shell.Explorer.2",
                                         Left=147,
                                         Top=60.75,
                                         Width=400,
                                         Height=400)
        control.Name = control_name[:31]
        browser = control.Object

    # set the chart aspect ratio to match the browser
    if control.Width > control.Height:
        chart.set_options("chart", {
             "height": "%d%%" % (100. * control.Height / control.Width)
         })
    else:
        chart.set_options("chart", {
             "width": "%d%%" % (100. * control.Width / control.Height)
         })

    # get the html and add the 'X-UA-Compatible' meta-tag
    soup = BeautifulSoup(chart.htmlcontent)
    metatag = soup.new_tag("meta")
    metatag.attrs["http-equiv"] = "X-UA-Compatible"
    metatag.attrs['content'] = "IE=edge"
    soup.head.insert(0, metatag)

    # write out the html for the browser to render
    fh = tempfile.NamedTemporaryFile("wt", suffix=".html", delete=False)
    filename = fh.name

    # clean up the file after 10 seconds to give the browser time to load
    def on_timer(timer_id, time):
        timer.kill_timer(timer_id)
        os.unlink(filename)
    timer.set_timer(10000, on_timer)

    fh.write(soup.prettify())
    fh.close()

    # navigate to the temporary file
    browser.Navigate("file://%s" % filename)

    return "[%s]" % control_name

@xl_func("string: object")
def hc_load_sample_data(name):
    """Loads Highcharts sample data from https://www.highcharts.com/samples/data/jsonp.php."""
    url = "https://www.highcharts.com/samples/data/jsonp.php?filename=%s.json&callback=?" % name
    return jsonp_loader(url)

@xl_func("object: var", auto_resize=True, category="Highcharts")
def hc_explode(data):
    """Explode a data object into an array.
    Caution: This may result in returning a lot of data to Excel.
    """
    return data


text:
python-highcharts
pywin32
bs4


text:
; To try this out, add this file to your existing pyxll.cfg
; file using the 'external_config' option.
;
; This code accompanies the blog post
; https://www.pyxll.com/blog/interactive-charts-in-excel-with-highcharts
[PYXLL]
modules =
    basic-line
    highcharts_xl
    line-time-series
    candlestick-and-volume

[PYTHON]
pythonpath =
    .
    ./highcharts
    ./highstocks


text:
"""
Highstock Demos
Two panes, candlestick and volume: http://www.highcharts.com/stock/demo/candlestick-and-volume

This code accompanies the blog post
https://www.pyxll.com/blog/interactive-charts-in-excel-with-highcharts
"""
from pyxll import xl_func
from highcharts import Highstock
from highcharts_xl import hc_plot

@xl_func
def hc_candlestick_and_volume(data, title, theme=None):
    H = Highstock()

    ohlc = []
    volume = []
    groupingUnits = [
        ['week', [1]],
        ['month', [1, 2, 3, 4, 6]]
    ]

    for i in range(len(data)):
        ohlc.append(
            [
            data[i][0], # the date
            data[i][1], # open
            data[i][2], # high
            data[i][3], # low
            data[i][4]  # close
            ]
            )
        volume.append(
            [
            data[i][0], # the date
            data[i][5]  # the volume
            ]
        )

    options = {
        'rangeSelector': {
            'selected': 1
        },

        'title': {
            'text': title
        },

        'yAxis': [{
            'labels': {
                'align': 'right',
                'x': -3
            },
            'title': {
                'text': 'OHLC'
            },
            'height': '60%',
            'lineWidth': 2
        }, {
            'labels': {
                'align': 'right',
                'x': -3
            },
            'title': {
                'text': 'Volume'
            },
            'top': '65%',
            'height': '35%',
            'offset': 0,
            'lineWidth': 2
        }],
    }

    H.add_data_set(ohlc, 'candlestick', 'OHLC', dataGrouping = {
        'units': groupingUnits
    })

    H.add_data_set(volume, 'column', 'Volume', yAxis = 1, dataGrouping = {
        'units': groupingUnits
    })

    H.set_dict_options(options)

    return hc_plot(H, title, theme)


text:
"""
Highcharts Demos
Basic line: https://www.highcharts.com/demo/line-basic

This code accompanies the blog post
https://www.pyxll.com/blog/interactive-charts-in-excel-with-highcharts
"""
from pyxll import xl_func
from highcharts_xl import hc_plot
from highcharts import Highchart

@xl_func("var[][] data, str title, str subtitle, str y_axis, str[] labels, str theme")
def hc_basic_line(data, title, subtitle=None, y_axis=None, labels=None, theme=None):
    H = Highchart()

    H.set_options("title", {
        "text": title
    })

    if subtitle:
        H.set_options("subtitle", {
            "text": subtitle
        })

    if y_axis:
        H.set_options("yAxis", {
            "title": {
                "text": y_axis
            }
        })

    H.set_options("legend", {
        "layout": "vertical",
        "align": "right",
        "verticalAlign": "middle"
    })

    # transform the data from a list of rows to a list of columns
    data_t = list(zip(*data))

    # Use the first column as the X axis
    x_axis = data_t[0]

    H.set_options("xAxis", {
        "categories": x_axis,
        "tickmarkPlacement": "on"
    })

    # And the remaining columns as the graph data
    if not labels:
        labels = ["series_%d" % i for i in range(len(data_t)-1)]

    for label, series in zip(labels, data_t[1:]):
        H.add_data_set(series, series_type='line', name=label)

    return hc_plot(H, title, theme)


text:
"""
Highcharts Demos
Time series, zoomable: http://www.highcharts.com/demo/line-time-series

This code accompanies the blog post
https://www.pyxll.com/blog/interactive-charts-in-excel-with-highcharts
"""
from pyxll import xl_func, get_type_converter
from highcharts_xl import hc_plot
from highcharts import Highchart

@xl_func
def hc_line_time_series(data, title, y_title=None, theme=None, xl_dates=False):
    """Plots a 2d array of times and data points."""
    # Convert Excel dates to datetimes if necessary
    if xl_dates:
        to_date = get_type_converter("var", "datetime")
        data = [(to_date(d), v) for d, v in data]

    H = Highchart()

    H.set_options('chart', {
        'zoomType': 'x'
    })

    H.set_options('xAxis', {
        'type': 'datetime'
    })

    H.set_options('yAxis', {
        'title': {
            'text': y_title or "Values"
        }
    })

    H.set_options('title', {
        'text': title or "Time Series"
    })

    H.set_options('legend', {
        'enabled': False
    })

    H.add_data_set(data, 'area')

    H.set_options('plotOptions', {
        'area': {
            'fillColor': {
                'linearGradient': { 'x1': 0, 'y1': 0, 'x2': 0, 'y2': 1},
                'stops': [
                    [0, "Highcharts.getOptions().colors[0]"],
                    [1, "Highcharts.Color(Highcharts.getOptions().colors[0]).setOpacity(0).get('rgba')"]
                ]},
            'marker': {
                'radius': 2
            },
            'lineWidth': 1,
            'states': {
                'hover': {
                    'lineWidth': 1
                }
            },
            'threshold': None
        }
    })

    return hc_plot(H, title, theme)
\


text:
"""
Decision tree example using scikit-learn.

This example shows how to fit a decision tree using scikit-learn
and make predictions in an Excel spreadsheet.

This code accompanies the blog post
https://www.pyxll.com/blog/machine-learning-in-excel-with-python
"""
from pyxll import xl_func, xl_app, async_call
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import train_test_split
import pandas as pd
import os

_zoo_classifications = {
    1: "mammal",
    2: "bird",
    3: "reptile",
    4: "fish",
    5: "amphibian",
    6: "insect",
    7: "mollusc"
}

@xl_func("float, int, int: object")
def ml_get_zoo_tree(train_size=0.75, max_depth=5, random_state=245245):
    # Load the zoo data
    dataset = pd.read_csv(os.path.join(os.path.dirname(__file__), "data", "zoo.csv"))

    # Drop the animal names since this is not a good feature to split the data on
    dataset = dataset.drop("animal_name", axis=1)

    # Split the data into a training and a testing set
    features = dataset.drop("class", axis=1)
    targets = dataset["class"]

    train_features, test_features, train_targets, test_targets = \
        train_test_split(features, targets, train_size=train_size, random_state=random_state)

    # Train the model
    tree = DecisionTreeClassifier(criterion="entropy", max_depth=max_depth)
    tree = tree.fit(train_features, train_targets)

    # Add the feature names to the tree for use in predict function
    tree._feature_names = features.columns

    return tree

@xl_func("object tree, dict<string, int> features: var")
def ml_zoo_predict(tree, features):
    # Convert the features dictionary into a DataFrame with a single row
    features = pd.DataFrame([features], columns=tree._feature_names)

    # Get the prediction from the model
    prediction = tree.predict(features)[0]

    # Update the image in Excel
    async_call(show_image_in_excel, prediction)

    return _zoo_classifications[prediction]

@xl_func("object tree, dict<string, int> features: series", auto_resize=True)
def ml_zoo_proba(tree, features):
    # Convert the features dictionary into a DataFrame with a single row
    features = pd.DataFrame([features], columns=tree._feature_names).astype(float)

    # Get the probabilities for each class
    proba = tree.predict_proba(features)[0]

    # Construct a pandas Series as the result
    index = [_zoo_classifications[c] for c in tree.classes_]
    return pd.Series(proba, index=index)

def show_image_in_excel(classification, figname="prediction_image"):
    """Plot a figure in Excel"""
    # Show the figure in Excel as a Picture object on the same sheet
    # the function is being called from.
    xl = xl_app()
    sheet = xl.ActiveSheet

    # if a picture with the same figname already exists then get the position
    # and size from the old picture and delete it.
    for old_picture in sheet.Pictures():
        if old_picture.Name == figname:
            height = old_picture.Height
            width = old_picture.Width
            top = old_picture.Top
            left = old_picture.Left
            old_picture.Delete()
            break
    else:
        # otherwise create a new image
        top_left = sheet.Cells(1, 1)
        top = top_left.Top
        left = top_left.Left
        width, height = 100, 100

    # insert the picture
    # Ref: http://msdn.microsoft.com/en-us/library/office/ff198302%28v=office.15%29.aspx
    filename = os.path.join(os.path.dirname(__file__), "images", _zoo_classifications[classification] + ".jpg")
    picture = sheet.Shapes.AddPicture(Filename=filename,
                                      LinkToFile=0,  # msoFalse
                                      SaveWithDocument=-1,  # msoTrue
                                      Left=left,
                                      Top=top,
                                      Width=width,
                                      Height=height)

    # set the name of the new picture so we can find it next time
    picture.Name = figname


text:
; To try this out, add this file to your existing pyxll.cfg
; file using the 'external_config' option.
;
; This code accompanies the blog post
; https://www.pyxll.com/blog/machine-learning-in-excel-with-python

[PYTHON]
pythonpath =
	./

[PYXLL]
modules =
	decision_tree


text:
1. Title: Zoo database

2. Source Information
   -- Creator: Richard Forsyth
   -- Donor: Richard S. Forsyth
             8 Grosvenor Avenue
             Mapperley Park
             Nottingham NG3 5DX
             0602-621676
   -- Date: 5/15/1990

3. Past Usage:
   -- None known other than what is shown in Forsyth's PC/BEAGLE User's Guide.

4. Relevant Information:
   -- A simple database containing 17 Boolean-valued attributes.  The "type"
      attribute appears to be the class attribute.  Here is a breakdown of
      which animals are in which type: (I find it unusual that there are
      2 instances of "frog" and one of "girl"!)

      Class# Set of animals:
      ====== ===============================================================
           1 (41) aardvark, antelope, bear, boar, buffalo, calf,
                  cavy, cheetah, deer, dolphin, elephant,
                  fruitbat, giraffe, girl, goat, gorilla, hamster,
                  hare, leopard, lion, lynx, mink, mole, mongoose,
                  opossum, oryx, platypus, polecat, pony,
                  porpoise, puma, pussycat, raccoon, reindeer,
                  seal, sealion, squirrel, vampire, vole, wallaby,wolf
           2 (20) chicken, crow, dove, duck, flamingo, gull, hawk,
                  kiwi, lark, ostrich, parakeet, penguin, pheasant,
                  rhea, skimmer, skua, sparrow, swan, vulture, wren
           3 (5)  pitviper, seasnake, slowworm, tortoise, tuatara
           4 (13) bass, carp, catfish, chub, dogfish, haddock,
                  herring, pike, piranha, seahorse, sole, stingray, tuna
           5 (4)  frog, frog, newt, toad
           6 (8)  flea, gnat, honeybee, housefly, ladybird, moth, termite, wasp
           7 (10) clam, crab, crayfish, lobster, octopus,
                  scorpion, seawasp, slug, starfish, worm

5. Number of Instances: 101

6. Number of Attributes: 18 (animal name, 15 Boolean attributes, 2 numerics)

7. Attribute Information: (name of attribute and type of value domain)
   1. animal name:      Unique for each instance
   2. hair		Boolean
   3. feathers		Boolean
   4. eggs		Boolean
   5. milk		Boolean
   6. airborne		Boolean
   7. aquatic		Boolean
   8. predator		Boolean
   9. toothed		Boolean
  10. backbone		Boolean
  11. breathes		Boolean
  12. venomous		Boolean
  13. fins		Boolean
  14. legs		Numeric (set of values: {0,2,4,5,6,8})
  15. tail		Boolean
  16. domestic		Boolean
  17. catsize		Boolean
  18. type		Numeric (integer values in range [1,7])

8. Missing Attribute Values: None

9. Class Distribution: Given above


text:
#! /usr/bin/env python3

import os
import sys
import json

import boto3

with open('credentials.json', 'r') as fd:
    credentials = json.loads(fd.read())

def main():

    s3 = boto3.client('s3',
                      endpoint_url=credentials['endpoint_url'],
                      aws_access_key_id=credentials['access_key'],
                      aws_secret_access_key=credentials['secret_key'])

    response = s3.list_buckets()
    for item in response['Buckets']:
        print(item['CreationDate'], item['Name'])

if __name__ == '__main__':
    main()


text:
#! /usr/bin/env python3

import os
import sys
from argparse import ArgumentParser
import json

import boto3

with open('credentials.json', 'r') as fd:
    credentials = json.loads(fd.read())

def main():

    parser = ArgumentParser(description='Creates a bucket')
    parser.add_argument('--bucket-name',
                        dest='bucket_name',
                        action='store',
                        required=True,
                        help='the name of the bucket to create')
    args = parser.parse_args()

    s3 = boto3.resource('s3',
                        endpoint_url=credentials['endpoint_url'],
                        aws_access_key_id=credentials['access_key'],
                        aws_secret_access_key=credentials['secret_key'])

    bucket = s3.Bucket(args.bucket_name)
    bucket.create()

if __name__ == '__main__':
    main()


text:
#! /usr/bin/env python3

import os
import sys
from argparse import ArgumentParser
import json

import boto3

with open('credentials.json', 'r') as fd:
    credentials = json.loads(fd.read())

def main():

    parser = ArgumentParser(description='Creates a bucket')
    parser.add_argument('--bucket-name',
                        dest='bucket_name',
                        action='store',
                        required=True,
                        help='the name of the bucket to create')
    args = parser.parse_args()

    s3 = boto3.resource('s3',
                        endpoint_url=credentials['endpoint_url'],
                        aws_access_key_id=credentials['access_key'],
                        aws_secret_access_key=credentials['secret_key'])

    bucket = s3.Bucket(args.bucket_name)
    bucket.delete()

if __name__ == '__main__':
    main()


text:
#! /usr/bin/env python3

import os
import sys
from argparse import ArgumentParser
import json

import boto3

with open('credentials.json', 'r') as fd:
    credentials = json.loads(fd.read())

def main():

    parser = ArgumentParser(description='Creates a bucket')
    parser.add_argument('--bucket-name',
                        dest='bucket_name',
                        action='store',
                        required=True,
                        help='the name of the bucket to create')
    parser.add_argument('--key-name',
                        dest='key_name',
                        action='store',
                        required=True,
                        help='the objects key name name')
    parser.add_argument('--data',
                        dest='data',
                        action='store',
                        required=True,
                        help='the object data')
    args = parser.parse_args()

    s3 = boto3.resource('s3',
                        endpoint_url=credentials['endpoint_url'],
                        aws_access_key_id=credentials['access_key'],
                        aws_secret_access_key=credentials['secret_key'])

    bucket = s3.Bucket(args.bucket_name)

    bucket.put_object(Bucket=args.bucket_name,
                      Key=args.key_name,
                      Body=args.data)

if __name__ == '__main__':
    main()


text:
#! /usr/bin/env python3

import os
import sys
from argparse import ArgumentParser
from io import BytesIO
import json

import boto3

with open('credentials.json', 'r') as fd:
    credentials = json.loads(fd.read())

def main():

    parser = ArgumentParser(description='Creates a bucket')
    parser.add_argument('--bucket-name',
                        dest='bucket_name',
                        action='store',
                        required=True,
                        help='the name of the bucket to create')
    parser.add_argument('--key-name',
                        dest='key_name',
                        action='store',
                        required=True,
                        help='the objects key name')
    parser.add_argument('--file-name',
                        dest='file_name',
                        action='store',
                        required=True,
                        help='the file to upload')
    args = parser.parse_args()

    s3 = boto3.resource('s3',
                        endpoint_url=credentials['endpoint_url'],
                        aws_access_key_id=credentials['access_key'],
                        aws_secret_access_key=credentials['secret_key'])

    bucket = s3.Bucket(args.bucket_name)

    data = BytesIO()

    bucket.download_fileobj(Fileobj=data, Key=args.key_name)

    print(data.getvalue())

if __name__ == '__main__':
    main()


text:
#! /usr/bin/env python3

import os
import sys
from argparse import ArgumentParser
import json

import boto3

with open('credentials.json', 'r') as fd:
    credentials = json.loads(fd.read())

def main():

    parser = ArgumentParser(description='Creates a bucket')
    parser.add_argument('--bucket-name',
                        dest='bucket_name',
                        action='store',
                        required=True,
                        help='the name of the bucket to create')
    parser.add_argument('--key-name',
                        dest='key_name',
                        action='store',
                        required=True,
                        help='the objects key name name')
    args = parser.parse_args()

    s3 = boto3.resource('s3',
                        endpoint_url=credentials['endpoint_url'],
                        aws_access_key_id=credentials['access_key'],
                        aws_secret_access_key=credentials['secret_key'])

    bucket = s3.Bucket(args.bucket_name)

    bucket.put_object(Bucket=args.bucket_name,
                      Key=args.key_name,
                      Body='')

if __name__ == '__main__':
    main()


text:
#! /usr/bin/env python3

import os
import sys
from argparse import ArgumentParser
import json

import boto3

with open('credentials.json', 'r') as fd:
    credentials = json.loads(fd.read())

def main():

    parser = ArgumentParser(description='Creates a bucket')
    parser.add_argument('--bucket-name',
                        dest='bucket_name',
                        action='store',
                        required=True,
                        help='the name of the bucket to create')
    parser.add_argument('--key-name',
                        dest='key_name',
                        action='store',
                        required=True,
                        help='the objects key name')
    parser.add_argument('--file-name',
                        dest='file_name',
                        action='store',
                        required=True,
                        help='the file to upload')
    args = parser.parse_args()

    s3 = boto3.resource('s3',
                        endpoint_url=credentials['endpoint_url'],
                        aws_access_key_id=credentials['access_key'],
                        aws_secret_access_key=credentials['secret_key'])

    bucket = s3.Bucket(args.bucket_name)

    bucket.download_file(Filename=args.file_name,
                         Key=args.key_name)

if __name__ == '__main__':
    main()


text:
#! /usr/bin/env python3

import os
import sys
from argparse import ArgumentParser
import json

import boto3

with open('credentials.json', 'r') as fd:
    credentials = json.loads(fd.read())

def main():

    parser = ArgumentParser(description='Creates a bucket')
    parser.add_argument('--bucket-name',
                        dest='bucket_name',
                        action='store',
                        required=True,
                        help='the name of the bucket to create')
    args = parser.parse_args()

    s3 = boto3.client('s3',
                      endpoint_url=credentials['endpoint_url'],
                      aws_access_key_id=credentials['access_key'],
                      aws_secret_access_key=credentials['secret_key'])

    response = s3.list_objects_v2(Bucket=args.bucket_name)
    for item in response['Contents']:
        print(item['Key'])

if __name__ == '__main__':
    main()


text:
#! /usr/bin/env python3

import os
import sys
from argparse import ArgumentParser
import json

import boto3

with open('credentials.json', 'r') as fd:
    credentials = json.loads(fd.read())

def main():

    parser = ArgumentParser(description='Creates a bucket')
    parser.add_argument('--bucket-name',
                        dest='bucket_name',
                        action='store',
                        required=True,
                        help='the name of the bucket to create')
    parser.add_argument('--key-name',
                        dest='key_name',
                        action='store',
                        required=True,
                        help='the objects key name')
    parser.add_argument('--file-name',
                        dest='file_name',
                        action='store',
                        required=True,
                        help='the file to upload')
    args = parser.parse_args()

    s3 = boto3.resource('s3',
                        endpoint_url=credentials['endpoint_url'],
                        aws_access_key_id=credentials['access_key'],
                        aws_secret_access_key=credentials['secret_key'])

    bucket = s3.Bucket(args.bucket_name)

    bucket.upload_file(Filename=args.file_name,
                       Key=args.key_name)

if __name__ == '__main__':
    main()


text:
# ceph-s3-examples
Examples of using boto3 to work with ceph


text:
#! /usr/bin/env python3

import os
import sys
from argparse import ArgumentParser
import json

import boto3

with open('credentials.json', 'r') as fd:
    credentials = json.loads(fd.read())

def main():

    parser = ArgumentParser(description='Creates a bucket')
    parser.add_argument('--bucket-name',
                        dest='bucket_name',
                        action='store',
                        required=True,
                        help='the name of the bucket to create')
    parser.add_argument('--key-name',
                        dest='key_name',
                        action='store',
                        required=True,
                        help='the objects key name name')
    args = parser.parse_args()

    s3 = boto3.client('s3',
                      endpoint_url=credentials['endpoint_url'],
                      aws_access_key_id=credentials['access_key'],
                      aws_secret_access_key=credentials['secret_key'])

    s3.delete_object(Bucket=args.bucket_name,
                     Key=args.key_name)

if __name__ == '__main__':
    main()


text:
body {
    font-family: Helvetica, "Helvetica Neue", Arial, "ヒラギノ角ゴ Pro W3", "Hiragino Kaku Gothic Pro", Osaka, "メイリオ", Meiryo, "ＭＳ Ｐゴシック", "MS PGothic", "小塚明朝 Pro L", serif;
}

.heading {
    text-align: center;
}
h2.heading, h3.heading {
    margin-top: -0.5em;
    font-weight: normal;
}

table {
    border-spacing: 0px;
    border-collapse: collapse;
}

td {
    border: 1px solid #dddddd;
    vertical-align: top;
}

tr:nth-child(even) {
    background: #ccc;
}

a, a:visited {
    color: #777;
    text-decoration: underline;
}

a:hover {
    background-color: #dFdFdF;
    text-decoration: none;
}

.ankiweb-download {
    border-radius: 4px;
    background: #3a3;
    border-color: #000;
    border-style: solid;
    border-width: 1px;
    color: #fff;
    display: inline-block;
    /*font: 20px/25px Helvetica Neue,Arial,"Lucida Grande",Sans-serif;
padding: 0.5em 1em;

    */
    padding: .1em;
    margin: 0;
    overflow: hidden;
    text-decoration: none;
    font-weight: bold;
    vertical-align: middle;
}

span.viz-field-span {
    margin-right: 0.5em;
}

/* For Nayr's Core5000 deck */
.kana, .reading {
    font-size: 50%;
    vertical-align: middle;
}

.target-words {
    color: red;
}
.target-words-romaji {
    color: blue;
}
.part-of-speech {
    color: green;
}
.target-words-meaning {
    color: purple;
}

repo_name: fasiha/fuzzy-anki

repo_language: JavaScript


text:
    <meta charset="UTF-8">
    <title>Fuzzy-Anki</title>

    <link rel="stylesheet" type="text/css" href="apkg.css">
    <link rel="stylesheet" type="text/css" href="c3.min.css">

Fuzzy-Anki Deck Browser

Turn your web browser into an Anki browser!

Browse notes and facts

Upload an Anki APKG file from your computer: <input type="file" id="ankiFile" name="files">.

    Experimental! Enter a URL of a public Anki deck: <input type="text" id="ankiURL" size="20"> <button id="ankiURLSubmit">Submit URL</button>. (Note to Ankiweb Shared Decks users: use deck's Download link, not the URL of the page describing it.)

 <input type="checkbox" id="showImage"> Show images? (If checked, Images will be loaded, but would cause long loading time for big files.)

View reviews

Get only <input type="text" id="sqliteLimit" size="5" value="150"> reviews. <input type="checkbox" id="sqliteRecent" checked> Most recent? (If unchecked, show the oldest.)

Upload your collection.anki2 file, usually found in <tt>Documents/Anki/<your name>/collection.anki2</tt> <input type="file" id="sqliteFile" name="files">


text:
var GLOBAL_CORS_PROXY = "http://cors-anywhere.herokuapp.com/";
var ankiSeparator = '\x1f';

// deckNotes contains the contents of any APKG decks uploaded. It is an array of
// objects with the following properties:
// - "name", a string
// - "fieldNames", an array of strings
// - "notes", an array of objects, each with properties corresponding to the
// entries of fieldNames.
var deckNotes;
var SQL;

// Huge props to http://stackoverflow.com/a/9507713/500207
function tabulate(datatable, columns, containerString) {
    var table = d3.select(containerString).append("table"),
        thead = table.append("thead"), tbody = table.append("tbody");

    // append the header row
    thead.append("tr")
        .selectAll("th")
        .data(columns)
        .enter()
        .append("th")
        .text(function(column) { return column; })
        .attr("class", function(d) { return 'field-' + d.replace(" ", "-"); });

    // create a row for each object in the data
    var rows = tbody.selectAll("tr").data(datatable).enter().append("tr");

    // create a cell in each row for each column
    var cells = rows.selectAll("td")
                    .data(
                         function(row) {
                             return columns.map(function(column) {
                                 return {column : column, value : row[column]};
                             });
                         })
                    .enter()
                    .append("td")
                    .html(function(d) { return d.value; })
                    .attr("class", function(d) {
                        return 'field-' + d.column.replace(" ", "-");
                    });

    return table;
}

function sqlToTable(uInt8ArraySQLdb) {
    var db = new SQL.Database(uInt8ArraySQLdb);

    // Decks table (for deck names)
    decks = db.exec("SELECT decks FROM col");
    // Could use parseJSON from jQuery here.
    decks = Function('return ' + decks[0].values[0][0])();

    // Models table (for field names)
    col = db.exec("SELECT models FROM col");
    // Could use parseJSON from jQuery here.
    var models = Function('return ' + col[0].values[0][0])();

    // Notes table, for raw facts that make up individual cards
    deckNotes = db.exec("SELECT mid,flds FROM notes");

    _.each(_.keys(models), function(key) {
        models[key].fields = _.pluck(models[key].flds, 'name');
    });

    var notesByModel =
        _.groupBy(deckNotes[0].values, function(row) { return row[0]; });

    deckNotes = _.map(notesByModel, function(notesArray, modelId) {
        var modelName = models[modelId].name;
        var fieldNames = models[modelId].fields;
        var notesArray = _.map(notesArray, function(note) {
            var fields = note[1].split(ankiSeparator);
            return arrayNamesToObj(fieldNames, fields);
        });
        return {name : modelName, notes : notesArray, fieldNames : fieldNames};
    });

    // Visualize!
    if (0 == specialDisplayHandlers()) {
        _.each(deckNotes, function(model, idx) {
            d3.select("#anki").append("h2").text(model.name);
            var deckId = "deck-" + idx;
            d3.select("#anki").append("div").attr("id", deckId);
            tabulate(model.notes, model.fieldNames, "#" + deckId);
            arrToCSV(model.notes, model.fieldNames, "Download CSV", d3.select("#" + deckId))
        });
    }
}

function parseImages(imageTable,unzip,filenames){
    var map = {};
    for (var prop in imageTable) {
      if (filenames.indexOf(prop) >= 0) {
        var file = unzip.decompress(prop);
        map[imageTable[prop]] = converterEngine (file);
      }
    }
    d3.selectAll("img")
      .attr("src", function(d,i) {
        //Some filenames may be encoded. Decode them beforehand.
        var key = decodeURI(this.src.split('/').pop());
        if (key in map){
          return "data:image/png;base64,"+map[key];
        }
          return this.src;
      });
}

function converterEngine (input) { // fn BLOB => Binary => Base64 ?
  // adopted from https://github.com/NYTimes/svg-crowbar/issues/16
    var uInt8Array = new Uint8Array(input),
        i = uInt8Array.length;
    var biStr = []; //new Array(i);
    while (i--) {
        biStr[i] = String.fromCharCode(uInt8Array[i]);
    }
    var base64 = window.btoa(biStr.join(''));
    return base64;
};

function ankiBinaryToTable(ankiArray, options) {
    var compressed = new Uint8Array(ankiArray);
    var unzip = new Zlib.Unzip(compressed);
    var filenames = unzip.getFilenames();
    var anki21Exists = filenames.indexOf("collection.anki21") >= 0;
    var sqliteFile = anki21Exists ? "collection.anki21" : "collection.anki2";
    if (filenames.indexOf(sqliteFile) >= 0) {
        var plain = unzip.decompress(sqliteFile);
        sqlToTable(plain);
        if (options && options.loadImage){
          if (filenames.indexOf("media") >= 0) {
              var plainmedia = unzip.decompress("media");
              var bb = new Blob([new Uint8Array(plainmedia)]);
              var f = new FileReader();
              f.onload = function(e) {
                parseImages(JSON.parse(e.target.result),unzip,filenames);
              };
              f.readAsText(bb);
          }
        }
    }
}

function ankiURLToTable(ankiURL, options, useCorsProxy, corsProxyURL) {
    if (typeof useCorsProxy === 'undefined') {
        useCorsProxy = false;
    }
    if (typeof corsProxyURL === 'undefined') {
        corsProxyURL = GLOBAL_CORS_PROXY;
    }

    var zipxhr = new XMLHttpRequest();
    zipxhr.open('GET', (useCorsProxy ? corsProxyURL : "") + ankiURL, true);
    zipxhr.responseType = 'arraybuffer';
    zipxhr.onload = function(e) { ankiBinaryToTable(this.response, options); };
    zipxhr.send();
}

function arrayNamesToObj(fields, values) {
    var obj = {};
    for (i in values) {
        obj[fields[i]] = values[i];
    }
    return obj;
}

function displayRevlogOutputOptions() {
    var ul = d3.select("body")
                 .append("div")
                 .attr("id", "reviews")
                 .append("div")
                 .attr("id", "reviews-options")
                 .append("ul")
                 .attr("id", "reviews-options-list");
    var tooMuch = 101;
    if (revlogTable.length > tooMuch) {
        ul.append('li')
            .attr("id", "tabulate-request")
            .append("button")
            .text("Tabulate " + revlogTable.length + " review" +
                  (revlogTable.length > 1 ? "s" : ""))
            .on("click", function() { tabulateReviews(); });

        ul.append('li')
            .attr("id", "export-request")
            .append("button")
            .text("Generate CSV spreadsheet")
            .on("click", function() { generateReviewsCSV(); });
    } else {
        tabulateReviews();
        generateReviewsCSV();
    }

    var viz = ul.append('li')
                  .attr("id", "viz-options");

    viz.append("button").text('Visualize performance').on("click", function() {
        var selectedFields = d3.selectAll("#viz-models-list > li.viz-model")
                                 .selectAll("input:checked");
        var config = selectedFields.map(function(mod) {
            mid = /[0-9]+/.exec(mod.parentNode.id)[0];
            fs = mod.map(function(sub) {
                var fnum = /field-([0-9]+)/.exec(sub.id)[1];
                return allModels[mid].flds[fnum].name;
            });
            return {modelID : mid, fieldNames : fs};
        });
        config = arrayNamesToObj(_.pluck(config, "modelID"),
                                 _.pluck(config, "fieldNames"));

        revlogVisualizeProgress(config, getSelectedDeckIDs());
    });

    var vizDecks =
        viz.append("ul").append('li').text("Select decks to analyze").append('ul').attr(
            "id", "viz-decks-list");
    var vizModels = viz.append("ul")
                        .append('li')
                        .text(
                             "Select fields for each model to display in plots")
                        .append('ul')
                        .attr("id", "viz-models-list");

    // Data: elements of decksReviewed (which are {deck IDs -> object})
    // TODO: enable visualization of unknown decks: .data(Object.keys(decksReviewed))
    var decksReviewedKeysAlphabetized =
        _.sortBy(Object.keys(_.omit(decksReviewed, null)), function(did) {
            return allDecks[did] ? allDecks[did].name : "zzzUnknown";
        });
    var vizDecksList = vizDecks.selectAll("li")
                           .data(decksReviewedKeysAlphabetized)
                           .enter()
                           .append("li")
                           .append('label')
                           .attr('for', function(d) { return 'viz-deck-' + d; })
                           .html(function(d, i) {
        var thisModels =
            _.filter(Object.keys(decksReviewed[d]).map(function(mid) {
            return d !== "null" ? allModels[mid].name : null;
        }), null);
        return '<input type="checkbox" checked id="viz-deck-' + d + '"> ' +
               (d !== "null" ? allDecks[d].name : "Unknown deck") +
               (thisModels.length > 0
                    ? " (contains model" +
                          (thisModels.length > 1 ? "s " : " ") +
                          thisModels.join(", ") + ")"
                    : "");
    });

    $('#viz-deck-null').attr("checked", false);

    $('#viz-decks-list input:checkbox')
        .click(function() { updateModelChoices(); });
    updateModelChoices();
}

function getSelectedDeckIDs() {
    var selectedDecks = _.pluck($('#viz-decks-list input:checked'), 'id');
    // In case the above is too fancy across browsers, this is equivalent:
    // `$.map($('#viz-decks-list input:checked'), function(x){return x.id;})`

    var selectedDeckIDs = selectedDecks.map(function(id) {
        return id !== "viz-deck-null" ? /[0-9]+/.exec(id)[0] : null;
    });
    return selectedDeckIDs;
}

function updateModelChoices() {
    var selectedDeckIDs = getSelectedDeckIDs();

    var modelIDs = _.union(_.flatten(_.map(selectedDeckIDs.map(function(did) {
        return decksReviewed[did];
    }), function(val) { return Object.keys(val); })));

    var vizModels = d3.select("#viz-models-list");
    var modelsData = vizModels.selectAll("li.viz-model")
                         .data(modelIDs, function(mid) { return mid; });
    // For an explanation of the CSS class 'viz-model' see
    // http://stackoverflow.com/a/25599142/500207

    modelsData.exit().remove();

    var vizModelsList =
        modelsData.enter()
            .append("li")
            .attr("id", function(mid) { return "viz-model-" + mid; })
            .text(
                 function(mid) {
                     return mid !== "null" ? allModels[mid].name
                                           : "Unknown model";
                 })
            /*.on("click", function(mid) {
                $('#viz-model-' + mid + '-list').slideToggle();
            })*/
            .classed("viz-model",
                     true).append("ul").append("li");

    var vizFields =
        vizModelsList.selectAll("span")
            .data(
                 function(d) {
                     return d !== "null"
                                ? (_.pluck(allModels[d].flds, 'name').map(
                                      function(name, idx) {
                                          return {
                                              name : name,
                                              modelId : d,
                                              total : allModels[d].flds.length
                                          };
                                      }))
                                : [];
                 })
            .enter()
            .append("span")
            .classed("viz-field-span", true)
            .append("label")
            .attr("for", function(d, i) {
                return 'viz-model-' + d.modelId + '-field-' + i;
            })
            .html(function(d, i) {
        return '<input type="checkbox" id="viz-model-' + d.modelId + '-field-' +
               i + '"> ' + d.name + (i + 1 < d.total ? ', ' : "");
    });
}

function arrToCSV(dataArray, fieldsArray, linkText, d3SelectionToAppend) {
    var csv = convert(dataArray, fieldsArray);
    var blob = new Blob([csv], {type : 'data:text/csv;charset=utf-8'});
    var url = URL.createObjectURL(blob);
    return d3SelectionToAppend.append("a")
        .attr("href", url)
        .text(linkText);
}

function generateReviewsCSV() {
    var d3Selection = arrToCSV(
        revlogTable,
        "dateString,ease,interval,lastInterval,timeToAnswer,noteSortKeyFact,deckName,modelName,lapses,\
reps,cardId,noteFactsJSON".split(','),
        "Download CSV", d3.select("#export-request").append("li").attr(
                            "id", "export-completed"));
    d3Selection.classed('csv-download', true);
}

function tabulateReviews() {
    tabulate(revlogTable,
             "date,ease,interval,lastInterval,timeToAnswer,noteSortKeyFact,deckName,modelName,lapses,\
reps,cardId,noteFactsJSON".split(','),
             "div#reviews");
}

// Note, this changes obj's parameters ("call by sharing") so the return value
// is purely a nicety: the object WILL be changed in the caller's scope.
function updateNestedObj(obj, outerKey, innerKey, innerVal) {
    if (!(outerKey in obj)) {
        obj[outerKey] = {};  // don't do {innerKey: innerKey} '_'
        obj[outerKey][innerKey] = innerVal;
    } else {
        if (!(innerKey in obj[outerKey])) {
            obj[outerKey][innerKey] = innerVal;
        }
    }
    return obj;
}

var sqliteGlobal;
var revlogTable;
var decksReviewed = {}, modelsReviewed = {}, allDecks, allModels;
function ankiSQLToRevlogTable(array, options) {
    if (typeof options === 'undefined') {
        options = {limit : 100, recent : true};
    }

    var sqliteBinary = new Uint8Array(array);
    var sqlite = new SQL.Database(sqliteBinary);
    sqliteGlobal = sqlite;

    // The deck name is in decks, and the field names are in models
    // which are JSON, and have to be handled outside SQL.
    var allModelsDecks = sqlite.exec('SELECT models,decks FROM col')[0].values[0];
    allModels = $.parseJSON(allModelsDecks[0]);
    allDecks = $.parseJSON(allModelsDecks[1]);

    // The reviews
    var query =
        'SELECT revlog.id, revlog.ease, revlog.ivl, revlog.lastIvl, revlog.time, notes.flds, notes.sfld, cards.id, cards.reps, cards.lapses, cards.did, notes.mid, cards.ord \
FROM revlog \
LEFT OUTER JOIN cards ON revlog.cid=cards.id \
LEFT OUTER JOIN notes ON cards.nid=notes.id \
ORDER BY revlog.id' +
        (options.recent ? " DESC " : "") +
        (options.limit && options.limit > 0 ? " LIMIT " + options.limit : "");
    var queryResultNames =
        "revId,ease,interval,lastInterval,timeToAnswer,noteFacts,noteSortKeyFact,cardId,reps,lapses,deckId,\
modelId,templateNum".split(',');

    // Run the query and convert the resulting array of arrays into an array of
    // objects
    revlogTable = sqlite.exec(query)[0].values;

    var unknownDeckString = "unknown deck";
    var unknownNoteString = "unknown note facts";
    var unknownModelString = "unknown model";
    // TODO add "Date of first review" field
    revlogTable = revlogTable.map(function(rev) {
        // First, convert this review from an array to an object
        rev = arrayNamesToObj(queryResultNames, rev);

        // Add deck name
        rev.deckName = rev.deckId ? allDecks[rev.deckId].name : unknownDeckString;

        // Convert facts string to a fact object
        var fieldNames =
            rev.modelId
                ? allModels[rev.modelId].flds.map(function(f) { return f.name; })
                : null;
        rev.noteFacts =
            rev.noteFacts ? arrayNamesToObj(fieldNames,
                                            rev.noteFacts.split(ankiSeparator))
                          : unknownNoteString;
        // Add model name
        rev.modelName =
            rev.modelId ? allModels[rev.modelId].name : unknownModelString;
        // delete rev.modelId;

        // Decks need to know what models are in them. decksReviewed is an
        // object of objects: what matters are the keys, at both levels, not the
        // values. TODO can this be done faster in SQL?
        updateNestedObj(decksReviewed, rev.deckId, rev.modelId, rev.modelName);
        // But let's also keep track of models in the same way, since we're lazy
        // FIXME
        updateNestedObj(modelsReviewed, rev.modelId, rev.deckId, rev.deckName);

        // Add review date
        rev.date = new Date(rev.revId);
        rev.dateString = rev.date.toString();

        // Add a JSON representation of facts
        rev.noteFactsJSON = typeof rev.noteFacts === "object"
                                ? JSON.stringify(rev.noteFacts)
                                : unknownNoteString;

        // Switch timeToAnswer from milliseconds to seconds
        rev.timeToAnswer /= 1000;

        return rev;
    });

    /*
    // decks and models that are only associated with reviews. Will this be
    // faster in sql.js or inside plain Javascript? TODO find out.
    var modelIDsReviewed = sqlite.exec(
                                      "SELECT DISTINCT notes.mid \
FROM revlog \
LEFT OUTER JOIN cards ON revlog.cid=cards.id \
LEFT OUTER JOIN notes ON cards.nid=notes.id")[0].values;
    modelsReviewed = modelIDsReviewed.map(function(mid) {
        return mid[0] ? allModels[mid[0]].name : unknownModelString;
    });
    modelIdToName =
        arrayNamesToObj(modelIDsReviewed.map(_.first), modelsReviewed);

    var deckIDsReviewed = sqlite.exec(
                                     "SELECT DISTINCT cards.did \
FROM revlog \
LEFT OUTER JOIN cards ON revlog.cid=cards.id")[0].values;
    decksReviewed = deckIDsReviewed.map(function(did) {
        return did[0] ? allDecks[did[0]].name : unknownDeckString;
    });
    deckIdToName = arrayNamesToObj(deckIDsReviewed.map(_.first), decksReviewed);
    */

    // Create div for results
    displayRevlogOutputOptions();
}

function reduceRevlogTable(deckIDsWanted) {
    deckIDsWanted = deckIDsWanted.map(function(i) { return parseInt(i); });

    // See if revlogTable is sorted ascending or descending by examining the
    // first two elements.
    // NB. This will fail if the SQL query isn't sorted by time!
    var oldestFirst = revlogTable[0].date < revlogTable[1].date;

    // We wanted to know whether the oldest came first or last because a key
    // element of this visualization is the date each note was learned.

    // Build the cardId-indexed array using reduce since it can reduce (left)
    // or reduceRight. Just accumulate the individual reviews. We don't need to
    // keep track of dates, or lapses, or total reps since the database gave us
    // that.
    var uniqueKeysSeenSoFar = 0;
    var temporalIndexToCardArray = [];
    var revDb;

    var reductionFunction = function(dbSoFar, rev, idx) {
        var key = rev.cardId;
        if (deckIDsWanted && deckIDsWanted.indexOf(rev.deckId) < 0) {
            return dbSoFar;
        }

        if (key in dbSoFar) {
            // Already seen this card ID
            dbSoFar[key].allRevlogs.push(rev);
        } else {
            // Fist time seeing this card ID
            dbSoFar[key] = {
                allRevlogs : [rev],
                reps : rev.reps,
                lapses : rev.lapses,
                cardId: rev.cardId,
                modelId : rev.modelId,
                dateLearned : rev.date,
                noteFacts : rev.noteFacts,
                temporalIndex : uniqueKeysSeenSoFar
            };
            temporalIndexToCardArray[uniqueKeysSeenSoFar] = key;
            uniqueKeysSeenSoFar++;
        }
        return dbSoFar;
    };

    // We know whether to reduce or reduceRight
    if (oldestFirst) {
        revDb = revlogTable.reduce(reductionFunction, {});
    } else {
        revDb = revlogTable.reduceRight(reductionFunction, {});
    }

    return {
        revDb : revDb,
        temporalIndexToCardArray : temporalIndexToCardArray
    };
}

function cardAndConfigToString(cardObj, config) {
    return config[cardObj.modelId].length > 0
               ? (config[cardObj.modelId]
                      .map(function(
                          factName) { return cardObj.noteFacts[factName]; })
                      .join(', '))
               : ("card ID: " + cardObj.cardId);
}

var revDb, temporalIndexToCardArray;
function revlogVisualizeProgress(configModelsFacts, deckIDsWanted) {
    // This function needs to take, as logical inputs, the decks and models to
    // limit the visualization to, plus a boolean operation AND or OR to combine
    // the two, and finally a way to display the pertinent facts about a card so
    // that cards are better-distinguished than card IDs (a long nunmber).
    if (typeof deckIDsWanted === undefined) {
        deckIDsWanted = [];
    }

    revDb = reduceRevlogTable(deckIDsWanted);
    temporalIndexToCardArray = revDb.temporalIndexToCardArray;
    revDb = revDb.revDb;

    // So now we've generated an object indexed by whatever keyFactId was chosen
    // (and potentially restricted to a deck/model) that tells us performance
    // details about each card. Sibling cards are currently treated as different
    // cards: TODO: allow user to select treating them as the same card.

    function appendC3Div(heading, text, id) {
        var newdiv = d3.select("#reviews").append("div");
        newdiv.append("h4").text(heading);
        newdiv.append('p').text(text);
        newdiv.append('div').attr("id", id);
        // d3.select("#reviews").append('div').attr("id", id);
    }

    appendC3Div("Performance since acquisition", "Number of lapses since \
card learned. Drag to pan, and mouse-weel to zoom.", "scatter-norm-rep-lapse");

    appendC3Div("Performance histogram",
                "Histogram of per-card performance, where ease of 1 is \
failure and all other eases are success.",
                "histogram");

    appendC3Div("Calendar view of acquisition",
                "Time series showing when cards were learned. \
Large circles indicate perfect performance, smaller circles indicate poorer \
performance. Zoomable and pannable.",
                "chart");

    appendC3Div("Scatter plot of lapses versus reps",
                "Lapses and reps are correlated with poor \
performance, so this scatter plot cannot be easily used for analysis.",
                "scatter-rep-lapse");

    //------------------------------------------------------------------------
    // Pass rate per unique card
    //------------------------------------------------------------------------
    // Generate the column-wise array of arrays that c3js wants
    var chartArr = _.map(revDb, function(val, key) {
        return [ val.dateLearned, 1 + val.temporalIndex ];
    });
    chartArr.unshift(['date', 'card index']);

    // Invoke the c3js method
    var chart = c3.generate({
        bindto : '#chart',
        data : {
                 x : 'date',
                 rows : chartArr,
                 onmouseover :
                     function(d, i) {
                         $('.c3-circle-' + d.index).css({
                             "stroke-width": 5
                         });
                     },
                 onmouseout :
                     function(d, i) {
                         $('.c3-circle-' + d.index).css({
                             "stroke-width": 1
                         });
                     }
               },
        axis : {
                 y : {label : {text : "Card index"}},
                 x : {
                     type : 'timeseries',
                     label : {text : "Date"},
                     tick : {rotate : 15,  count: 50, format : '%Y-%m-%d %I:%M'},
                     height : 40,
                 }
               },
        tooltip : {
                    format : {
                        value : function(value, ratio, id) {
                            // value: 1-index!
                            var key = temporalIndexToCardArray[value-1];
                            var str = cardAndConfigToString(revDb[key],
                                                            configModelsFacts);
                            var reps = revDb[key].reps;
                            var lapses = revDb[key].lapses;
                            return str +
                                   " (#" + (value - 1 + 1) + ", " + (reps-lapses) +
                                   '/' + reps + " reps passed)";
                        }
                    }
                  },
        legend : {show : false},
        zoom : {
                 enabled : true,
                 extent : [
                     1,
                     2
                 ]
               },  // default is [1,10] doesn't provide enough zoooooom
        point : {
            focus :
                {expand : {enabled : false}}
        }  // don't expand a point on focus
    });

    // Make the radius and opacity of each data circle depend on the pass rate
    var grader =
        function(dbentry) { return 1 - dbentry.lapses / dbentry.reps; };
    var worstRate = grader(_.min(revDb, grader));
    var scaleRadius = d3.scale.linear().domain([ worstRate - .005, 1 ]).range([ 2, 45 ]);
    var scaleOpacity =
        d3.scale.pow().exponent(-17).domain([ worstRate, 1 ]).range([ 1, 0.05 ]);

    // The following helps smooth out the diversity of radii and opacities by
    // putting more slope in the linear scale where there's more mass in the
    // histogram, so when there's lots of things with about the same value,
    // they'll have more different radii/opacities than they would otherwise. It
    // looks good, but it depends on the user's data, and requires some
    // automatic histogram analysis: TODO.
    if (false) {
        var lin = d3.scale.linear().domain([ 0, 1 ]).range(scaleRadius.range());
        scaleRadius =
            d3.scale.linear()
                .domain([ worstRate, .85, .93, .96, 1 ])
                .range([ lin(0), lin(.2), lin(.8), lin(.99), lin(1) ]);
        lin = d3.scale.linear().domain([ 0, 1 ]).range(scaleOpacity.range());
        scaleOpacity =
            d3.scale.linear()
                .domain([ worstRate, .85, .93, .96, 1 ])
                .range([ lin(0), lin(.2), lin(.8), lin(.99), lin(1) ]);
    }

    temporalIndexToCardArray.forEach(function(value, idx) {
        var dbentry = revDb[temporalIndexToCardArray[idx]];
        var rate = grader(dbentry);
        // if (idx>=557) {debugger;}
        d3.select('.c3-circle-' + idx).attr({
            'r' : scaleRadius(rate),
            //'fill-opacity' : 0,
            //'fill' : 'none',
            'stroke-opacity' : scaleOpacity(rate)
        });
    });
    $('.c3-circle').css({stroke : 'rgb(31,119,180)', fill: "none", "fill-opacity": 0});

    //------------------------------------------------------------------------
    // Histogram of pass rates
    //------------------------------------------------------------------------
    // High to low, then reverse, to make sure 1.01 and 1 have no roundoff.
    // Include 1.01 to capture 1 in its own bin
    var binDistance = 0.01;
    var histEdges = _.range(1.01, Math.floor(worstRate * 100) / 100,
                            -binDistance).reverse();

    var histData = d3.layout.histogram().bins(histEdges)(_.map(revDb, grader));
    var normalizeHistToPercent = 1 / (temporalIndexToCardArray.length);
    var chartHistData =
        _.map(histData, function(bar) { return [ bar.x, bar.y ]; });
    chartHistData.unshift([ 'x', 'frequency' ]);
    var hist = c3.generate({
        bindto : '#histogram',
        data : {x : 'x', rows : chartHistData, type : "bar"},
        bar : {width : {ratio : .95}},
        axis : {
                 y : {label : {text : "Number of cards"}},
                 x : {
                     label : {text : "Pass rate"},
                     tick : {format : d3.format('.2p')}

                 }
               },
        tooltip : {
                    format : {
                        value : function(value, ratio, id) {
                            return value + ' cards (' +
                                   d3.format('.3p')(value *
                                                    normalizeHistToPercent) +
                                   ' of cards)';
                        }
                    }
                  },
        legend : {show : false}
    });

    //-----------------
    // Time to failure plots
    //--------------------
    var unitRandom = function() { return (Math.random() - 0.5) * .5; };
    var lapsesReps = temporalIndexToCardArray.map(
        function(key, idx){return [ revDb[key].lapses + unitRandom(), revDb[key].reps + unitRandom()]});
    lapsesReps.unshift(['lapses', 'reps']);
    var lapsesRepsChart = c3.generate({
        bindto : '#scatter-rep-lapse',
        data : {x : 'reps', rows : lapsesReps, type : "scatter"},
        axis : {
                 x : {label : {text : "# reps, integer with jitter"}, tick : {fit : false}},
                 y : {label : {text : "# lapses, integer with jitter"}}
               },
        legend : {show : false}
    });

    //-----------
    // Normalized
    //-----------
    var current = new Date().getTime();
    var dayDiff = function(initial) {
        return (current - initial.getTime()) / (1000 * 3600 * 24);
    };
    jitteredTimeToCard = {};
    var lapsesTime = temporalIndexToCardArray.map(function(key, idx) {
        var jitteredTime = dayDiff(revDb[key].dateLearned) + unitRandom();
        jitteredTimeToCard[jitteredTime] = key;
        return [ revDb[key].lapses + unitRandom(), jitteredTime ];
    });
    lapsesTime.unshift([ 'lapses', 'daysKnown' ]);

    /*
    var lapsesTimesTranspose = [];
    for (var inputCol = 0;inputCol < lapsesTime[0].length; inputCol++) {
        lapsesTimesTranspose[inputCol] = [];
        for (var inputRow = 0; inputRow < lapsesTime.length; inputRow++) {
            lapsesTimesTranspose[inputCol][inputRow] = lapsesTime[inputRow][inputCol];
        }
    }
    */ /*data --> columns : lapsesTimesTranspose*/

    var lapsesDaysChart = c3.generate({
        bindto : '#scatter-norm-rep-lapse',
        data : {x : 'daysKnown', rows : lapsesTime, type : "scatter"},
        axis : {
                 x : {
                       label : {text : "days known, with jitter"},
                       tick : {fit : false}
                     },
                 y : {label : {text : "# lapses, with jitter"}}
               },
        legend : {show : false},
        tooltip :
            {
              contents :
                  function(d, defaultTitleFormat, defaultValueFormat, color) {
                      var key = jitteredTimeToCard[d[0].x];
                      var str = cardAndConfigToString(revDb[key],
                                                            configModelsFacts);
                      this.config.tooltip_format_title = function(d) {
                          return "Known for " + d3.round(d) + " days (" + str +
                                 ")";
                      };
                      this.config.tooltip_format_value =
                          function(value, ratio, id) {
                              return d3.round(value) + " (" + str + ")";
                      };
                      var retval =
                          this.getTooltipContent
                              ? this.getTooltipContent(d, [], [], color)
                              : '';
                      return retval;
                  },
              format : {
                  title :
                      function(d) {
                          return "Known for " + d3.round(d) + " days";
                      },
                  name :
                      function(id) {
                          if (id === "lapses") {
                              return "Lapses";
                          }
                          return "Card key";
                      },
                  value :
                      function(value, ratio, id) {
                          if (id === "lapses") {
                              return d3.round(value);
                          }
                          return temporalIndexToCardArray[value];
                      }
              }
            },
        zoom : {enabled : true, extent : [ 1, 2 ]}
    });
}

// Lifted from
// https://github.com/matteofigus/nice-json2csv/blob/master/lib/nice-json2csv.js
// (MIT License)
function fixInput(parameter) {
    if (parameter && parameter.length == undefined &&
        _.keys(parameter).length > 0)
        parameter = [parameter];  // data is a json object instead of an array
                                  // of json objects

    return parameter;
}
function getColumns(data) {
    var columns = [];

    for (var i = 0; i < data.length; i++)
        columns = _.union(columns, _.keys(data[i]));

    return columns;
}
function convertToCsv(data) {
    return JSON.stringify(data)
        .replace(/],\[/g, '\n')
        .replace(/]]/g, '')
        .replace(/\[\[/g, '')
        .replace(/\\"/g, '""');
}
function convert(data, headers, suppressHeader) {
    if (!_.isBoolean(suppressHeader)) suppressHeader = false;

    data = fixInput(data);

    if (data == null || data.length == 0) {
        return "";
    }

    var columns = headers ? ((typeof headers == 'string') ? [headers] : headers)
                          : getColumns(data);

    var rows = [];

    if (!suppressHeader) {
        rows.push(columns);
    }

    for (var i = 0; i < data.length; i++) {
        var row = [];
        _.forEach(columns, function(column) {
            var value =
                typeof data[i][column] == "object" && data[i][column] &&
                    "[Object]" ||
                typeof data[i][column] == "number" && String(data[i][column]) ||
                data[i][column] || "";
            row.push(value);
        });
        rows.push(row);
    }

    return convertToCsv(rows);
}

$(document).ready(function() {
    initSqlJs({locateFile: filename => filename}).then(function(localSQL){
        SQL = localSQL;
        readySetup();
    });
});
function readySetup() {
    var options = {};
    var setOptionsImageLoad = function(){
        options.loadImage = $('input#showImage').is(':checked');
        return options;
    }
    var eventHandleToTable = function(event) {
        event.stopPropagation();
        event.preventDefault();
        var f = event.target.files[0];
        if (!f) {
            f = event.dataTransfer.files[0];
        }
        // console.log(f.name);

        var reader = new FileReader();
        if ("function" in event.data) {
            reader.onload =
                function(e) { event.data.function(e.target.result); };
        } else {
            reader.onload = function(e) { ankiBinaryToTable(e.target.result, setOptionsImageLoad()); };
        }
        /* // If the callback doesn't need the File object, just use the above.
        reader.onload = (function(theFile) {
            return function(e) {
                console.log(theFile.name);
                ankiBinaryToTable(e.target.result);
            };
        })(f);
        */
        reader.readAsArrayBuffer(f);
    };

    // Deck browser
    $("#ankiFile")
        .change({
                  "function" :
                      function(data) {
                          ankiBinaryToTable(data, setOptionsImageLoad());
                      }
                }, eventHandleToTable);
    $("#ankiURLSubmit").click(function(event) {
        ankiURLToTable($("#ankiURL").val(), setOptionsImageLoad(), true);
        $("#ankiURL").val('');
    });

    // Review browser
    $("#sqliteFile")
        .change({
                  "function" :
                      function(data) {
                          ankiSQLToRevlogTable(data, {
                              limit : parseInt($('input#sqliteLimit').val()),
                              recent : $('input#sqliteRecent').is(':checked')
                          });
                      },
                },
                eventHandleToTable);

    // Only for local development
    // ankiURLToTable('/n.apkg');
};

/**
* Hook that modifies Nayr's Japanese Core5000 Anki deck
* (see https://ankiweb.net/shared/info/631662071)
*
* @param {Array} deckNotes - array of Anki Notes from the above deck
* @param {String[]} deckFields - names of the fields of the Notes
* @return {Array} an updated version of deckNotes
*
* Each Note object containing properties Expression, Meaning, Reading, English
* Translation, Word, Frequency Order, and Sound.
*
* Kana in the "Reading" field will be changed from "[kana]" to being wrapped in
*<span> tags. And each of the items in the "Word" field, which contains the
*Japanese word, its reading in roumaji (Latin characters), one or more
*parts-of-speech, and English translations, will be encased in <span> tags
*(ideally these would be their own independent fields, but some rows have more
*than one part-of-speech).
*/
function core5000Modify(deckNotes, deckFields, deckName) {
    d3.select("body").append("div").attr("id", "core5000");
    d3.select("#core5000").append("h2").text(deckName);
    var divForLink = d3.select("#core5000").append("p");

    //------------------------------------------------------------
    // Variables and functions to help deal with the "Word" column
    //------------------------------------------------------------
    // Parts of speech abbreviations
    var abbreviations =
        "adn.,adv.,aux.,conj.,cp.,i-adj.,interj.,n.,na-adj.,num.,p.,p. \
case,p. conj.,p. disc.,pron.,v.,suffix,prefix".split(',');
    var abbreviationsOr = abbreviations.join("|").replace(/\./g, '\\.');

    // The basic structure of the "Word" column is:
    //
    // 1. some kanji or kana, plus other random things like commas, parentheses,
    // both ascii and full-width.
    // 2. Some roumaji
    // 3. One or more parts of speech, using the above abbreviations
    // 4. English translations.
    //
    // The following three strings will be the regexps that match #1--#3.
    // They've been carefully chosen to work with wrinkles in the database,
    // e.g., more than one of the above four-step sequences in a single row,
    // multiple adjacent parts-of-speech, or multiple
    // part-of-speech-and-translation pairs. All these strings intended to
    // become regexps will go through XRegExp, which expands out the
    // Han/Katakana/Hiragana groups.
    var kanaKanjiWordRegexp = '([^a-z]+)';
    var romajiRegexp = '([a-z\\s,\\-()’]+)';
    var partOfSpeechRegexp = '((?: |,|' + abbreviationsOr + ')+)';

    // Break up a string containing one {kanji/kana + roumaji + part-of-speech +
    // translations} sequence. The critical idea in this function is to split
    // the input string between part-of-speech-abbreviations, and do some
    // processing on that to handle two edge cases:
    //
    // 1. "いろいろ iroiro adv., na-adj. various" <-- more than one adjacent
    //     part-of-speech abbreviation separated by a comma
    // 2. "余り amari adv. the rest n. (not) much" <-- more than one
    //     part-of-speech/translation pairs.
    //
    // It handles both these cases by splitting the string into an array along
    // (and including) part-of-speech-abbreviation boundaries. To handle edge
    // case 1 above, it finds elements of the resulting array that are
    // between part-of-speech abbreviations but which are
    // whitespace/punctuation, and merges those elements into a single
    // "part-of-speech" element.
    //
    // Then it builds an array of parts-of-speech and a matching array of
    // translations. This handles case 2 above. These two arrays, as well as the
    // kanji/kana and roumaji, are returned as an object.
    function bar(seqString) {
        // How much hackier can we get :)
        if (0 ==
            seqString.localeCompare(
                "（お）姉さん(o)-nee-san n. elder sister")) {
            // Add space between Japanese and reading
            seqString = "（お）姉さん (o)-nee-san n. elder sister";
        } else if (0 ==
                   seqString.localeCompare(
                       "相変わらず ai-kawara zu adv as ever, as usual, the \
same, as before [always]")) {
            // Add dot to "adv", completing the abbreviation instead of adding
            // another abbreviation which might trigger elsewhere
            seqString =
                "相変わらず ai-kawara zu adv. as ever, as usual, the same, as \
before [always]";
        } else if (0 == seqString.localeCompare("ごと-goto suffix every")) {
            seqString = "ごと goto suffix every";
        } else if (0 == seqString.localeCompare("家 uchi n house, home")) {
            seqString = "家 uchi n. house, home";
        }

        var arr = seqString.split(XRegExp('(' + abbreviationsOr + ')'));

        var isAbbreviation = arr.map(
            function(x) { return abbreviations.indexOf(x) >= 0 ? 1 : 0; });
        var isWhitePunctuation =
            arr.map(function(x) { return x.match(/^[\s,]*$/) ? 1 : 0; });
        var isAbbrOrWhitePunct = isAbbreviation.map(
            function(x, i) { return x + isWhitePunctuation[i]; });

        // combineJunk will find [..., "adv.", ",", "na-adj.", ...] and splice
        // it into [..., "adv., na-adj.", ...].
        var tmp = combineJunk(isAbbrOrWhitePunct, arr);
        arr = tmp.data_array;
        isAbbrOrWhitePunct = tmp.indicator_array;
        // Updated arr and isAbbrOrWhitePunct. We need the latter to build the
        // return object.

        // Part-of-speech array and translation array, which will go itno the
        // return object. We rely on each part-of-speech element in arr to be
        // followed by a translation. So far, this happens.
        var pos = [];
        var translation = [];
        arr.map(function(x, i) {
            if (isAbbrOrWhitePunct[i]) {
                pos.push(x);
                translation.push(arr[i + 1]);
            }
        });

        // Grab the initial kanji/kana as well as the roumaji. String.match()
        // will return a three-element array here: the total match, and the two
        // groups corresponding to the two regexps.
        var kanaKanjiMatch =
            seqString.match(XRegExp(kanaKanjiWordRegexp + ' ' + romajiRegexp +
                                    ' ' + partOfSpeechRegexp));

        if (0==seqString.localeCompare("Oa oobii n. OB (old boy), alumnus")) {
            return {
                pos : pos,
                translation : translation,
                word : "OB",
                romaji : "oobii"
            };
        }
        return {
            pos : pos,
            translation : translation,
            word : kanaKanjiMatch[1],
            romaji : kanaKanjiMatch[2]
        };
    };

    function combineJunk(indicator_array, data_array) {
        var i = 1;
        while (i < indicator_array.length) {
            if (indicator_array[i] == indicator_array[i - 1] &&
                indicator_array[i] > 0) {
                indicator_array.splice(i - 1, 2, 1);
                data_array.splice(i - 1, 2, data_array[i - 1] + data_array[i]);
            } else {
                i++;
            }
        }
        return {data_array : data_array, indicator_array : indicator_array};
    }

    // Get rid of   and such. It'll mess up my regexping.
    function decodeHtml(html) {
        var txt = document.createElement("textarea");
        txt.innerHTML = html;
        return txt.value;
    }

    var wordColumnReplace = function(s) {
        if (s.search("&") >= 0) {
            s = decodeHtml(s);
        }

        var arr = s.split("<div>");

        return arr.map(function(s) {
            var decomp = bar(s);
            var posTrans = decomp.pos.map(function(pos, i) {
                return '<span class="part-of-speech">' + pos +
                       '</span> <span class="target-words-meaning">' +
                       decomp.translation[i] + '</span>';
            }).join(" ");
            return '<span class="target-words">' + decomp.word +
                   '</span> <span class="target-words-romaji">' +
                   decomp.romaji + "</span> " + posTrans;
        }).join("<div>");
    };

    //--------------------------------------------
    // Variable for Reading column cleanup of kana
    //--------------------------------------------
    var kanaRegexp = XRegExp('\\[([\\p{Hiragana}\\p{Katakana}]+)\\]', 'g');

    //-----------------
    // Complete cleanup
    //-----------------
    deckNotes.map(function(note, loc, arr) {
        // Again, how much hackier can you get :)
        if (0 == note.Reading.localeCompare("この 単語[たんご]はどういう 意味[いみ]ですか。")) {
            note.Word = "語 go n. word; language";
        }

        // Break up Word column into its four separate components
        note.Word = wordColumnReplace(note.Word);

        // Replace [kana] with spans
        note.Reading = note.Reading.replace(kanaRegexp,
                                            function(match, kana, offset, str) {
            return '<span class="reading kana">' + kana + '</span>';
        });

        return note;
    });

    //-------------------------
    // Visualization and return
    //-------------------------
    arrToCSV(deckNotes, deckFields, "Download Nyar's Core5k CSV", divForLink);

    tabulate(deckNotes, deckFields, "#core5000");

    // Instead of setting the styles of thousands of <td> tags individually,
    // just slash on a CSS tag to the DOM.
    d3.select("head").insert("style", ":first-child").text(
        "#core5000 th.field-Meaning, #core5000 th.field-Sound {font-size: 10%}\
#core5000 th.field-Frequency-Order {font-size:50%}\
#core5000 td.field-Expression, #core5000 td.field-Reading {font-size: 150%}\
#core5000 td.field-English-Translation, #core5000  td.field-Word {font-size: 75%}");

    return deckNotes;
}

function specialDisplayHandlers() {
    if (false) {
        modifiedDeckNotes = _.map(
            _.filter(deckNotes, function(model) {
                return 0 ==
                       "Nayr's Japanese Core5000".localeCompare(model.name);
            }),
            function(model) {
                return core5000Modify(model.notes, model.fieldNames,
                                      model.name);
            });
        if (modifiedDeckNotes.length > 0) {
            return 1;
        }
    }
    return 0;
}

var summer = function(arr) {
    return _.reduce(arr, function(memo, num) { return memo + num; }, 0);
};
var mean = function(arr) { return summer(arr) / arr.length; };