# -*- coding: utf-8 -*-
"Vision data generated by scrapers/awty.py and then edited by hand"
from data.vision import *

# Data imported from Rodrigo Benenson's "Who is the Best at X / Are we there
# yet?" (https://rodrigob.github.io/are_we_there_yet/build/#about)

msrc21_pc = image_classification.metric("MSRC-21 image semantic labelling (per-class)", "http://jamie.shotton.org/work/data.html", scale=correct_percent)
msrc21_pp = image_classification.metric("MSRC-21 image semantic labelling (per-pixel)", "http://jamie.shotton.org/work/data.html", scale=correct_percent)


cifar100 = image_classification.metric("CIFAR-100 Image Recognition", "http://https://www.cs.toronto.edu/~kriz/cifar.html", scale=correct_percent)

cifar10 = image_classification.metric("CIFAR-10 Image Recognition", "http://https://www.cs.toronto.edu/~kriz/cifar.html", scale=correct_percent, target=94, target_source="http://karpathy.github.io/2011/04/27/manually-classifying-cifar10/")

svhn = image_classification.metric("Street View House Numbers (SVHN)", "http://ufldl.stanford.edu/housenumbers/", scale=error_percent, target=2.0, target_source="http://ufldl.stanford.edu/housenumbers/nips2011_housenumbers.pdf")

# We declare MNIST solved because the gap between best performance and human performance appears to be less than the uncertainty in human performance
mnist = image_classification.metric("MNIST handwritten digit recognition", "http://yann.lecun.com/exdb/mnist/", scale=error_percent, target=0.2, target_source="http://people.idsia.ch/~juergen/superhumanpatternrecognition.html", solved=True)

stl10 = image_classification.metric("STL-10 Image Recognition", "https://cs.stanford.edu/~acoates/stl10/", scale=correct_percent)


leeds_sport_poses = image_classification.metric("Leeds Sport Poses")


# Data not in the AWTY scrape at all
# This awty URL broken
mnist.measure(date(2013,2,28), 0.52, 'COSFIRE', 'http://www.cs.rug.nl/~george/articles/PAMI2013.pdf', papername='Trainable COSFIRE Filters for Keypoint Detection and Pattern Recognition')
mnist.measure(None, 0.38, 'Fitnet-LSUV-SVM', url='http://arxiv.org/abs/1511.06422', papername='All you need is a good init', uncertainty=0.0, venue='ICLR 2015')
# -------------------------------------------------
# additional, newer data on the AWTY problems
# -------------------------------------------------

mnist.measure(None, 100 - 99.31, "HyperGAN 100", url="https://arxiv.org/pdf/1901.11058.pdf")
cifar10.measure(None, 77.71, "HyperGAN 100", url="https://arxiv.org/pdf/1901.11058.pdf")


cifar100.measure(None, 100 - 22.71, "ResNet-1001", url="https://arxiv.org/pdf/1603.05027", uncertainty=0.22)
cifar10.measure(None, 100 - 4.62, "ResNet-1001", url="https://arxiv.org/pdf/1603.05027", uncertainty=0.20)
cifar100.measure(None, 69.0, "NiN+Superclass+CDJ", url="https://arxiv.org/abs/1706.02003")
stl10.measure(None, 77.79, u"CC-GAN²", url="https://arxiv.org/abs/1611.06430v1", uncertainty=0.8)
cifar10.measure(None, 100 - 5.62, "ResNet+ELU", url="https://arxiv.org/pdf/1604.04112.pdf")
cifar100.measure(None, 100 - 26.55, "ResNet+ELU", url="https://arxiv.org/pdf/1604.04112.pdf")

cifar10.measure(None, 94.6, "Evolution", url="https://arxiv.org/pdf/1703.01041.pdf")
cifar10.measure(None, 95.6, "Evolution ensemble", url="https://arxiv.org/pdf/1703.01041.pdf")
cifar100.measure(None, 77.0, "Evolution", url="https://arxiv.org/pdf/1703.01041.pdf") 

cifar10.measure(None, 96.35, "Neural Architecture Search", url="https://arxiv.org/abs/1611.01578v2", venue="ICLR 2017")
cifar10.measure(None, 94.6, "RL+NT", url="https://arxiv.org/pdf/1707.04873.pdf")

cifar10.measure(None, 94.4, "Deep Complex", "https://arxiv.org/abs/1705.09792v2")
cifar100.measure(None, 72.91, "Deep Complex", "https://arxiv.org/abs/1705.09792v2")
svhn.measure(None, 3.3, "Deep Complex", "https://arxiv.org/abs/1705.09792v2")


cifar10.measure(None, 97.11, "ENAS", url="https://arxiv.org/pdf/1802.03268.pdf")


# ---------------------------------------------------------------------------------
# Originally generated by scrapers/awty.py
# Algorithm names hurriedly edited to improve graphs; please fix them where they
# don't match what's used in the literature

# Handling 'STL-10' classification_datasets_results.html#53544c2d3130
stl10.measure(None, 74.33, 'SWWAE', url='http://arxiv.org/abs/1506.02351', papername='Stacked What-Where Auto-encoders', uncertainty=0.0, venue='arXiv 2015', notes='')
stl10.measure(None, 74.1, 'Convolutional Clustering', url='http://arxiv.org/abs/1511.06241', papername='Convolutional Clustering for Unsupervised Learning', uncertainty=0.0, venue='arXiv 2015', notes='3 layers + multi dict. With 2 layers, reaches 71.4%')
stl10.measure(None, 73.15, 'Deep Representation Learning with Target Coding', url='http://personal.ie.cuhk.edu.hk/~ccloy/files/aaai_2015_target_coding.pdf', papername='Deep Representation Learning with Target Coding', uncertainty=0.0, venue='AAAI 2015', notes='')
stl10.measure(None, 72.8, 'Discriminative Unsupervised Feature Learning with Convolutional Neural Networks', url='http://papers.nips.cc/paper/5548-discriminative-unsupervised-feature-learning-with-convolutional-neural-networks.pdf', papername='Discriminative Unsupervised Feature Learning with Convolutional Neural Networks', uncertainty=0.0, venue='NIPS 2014', notes='Unsupervised feature learning + linear SVM')
stl10.measure(None, 70.2, 'An Analysis of Unsupervised Pre-training in Light of Recent Advances', url='http://arxiv.org/abs/1412.6597', papername='An Analysis of Unsupervised Pre-training in Light of Recent Advances', uncertainty=0.0, venue='ICLR 2015', notes='Unsupervised pre-training, with supervised fine-tuning. Uses dropout and data-augmentation.')
stl10.measure(None, 70.1, 'Multi-Task Bayesian Optimization', url='http://hips.seas.harvard.edu/files/swersky-multi-nips-2013.pdf', papername='Multi-Task Bayesian Optimization', uncertainty=0.0, venue='NIPS 2013', notes='Also uses CIFAR-10 training data')
stl10.measure(None, 68.23, 'C-SVDDNet', url='http://arxiv.org/abs/1412.7259', papername='C-SVDDNet: An Effective Single-Layer Network for Unsupervised Feature Learning', uncertainty=0.0, venue='arXiv 2014', notes='')
stl10.measure(None, 68.0, 'DFF Committees', url='http://arxiv.org/abs/1406.5947', papername='Committees of deep feedforward networks trained with few data', uncertainty=0.0, venue='arXiv 2014', notes='')
stl10.measure(None, 67.9, 'Nonnegativity Constraints ', url='http://jmlr.org/proceedings/papers/v32/line14.pdf', papername='Stable and Efficient Representation Learning with Nonnegativity Constraints ', uncertainty=0.0, venue='ICML 2014', notes=u'3-layers + multi-dict. 5 \xb1 0.5 with 3-layers only. 6 \xb1 0.6 with 1-layers only.')
stl10.measure(None, 64.5, 'RGB-D Based Object Recognition', url='http://homes.cs.washington.edu/~lfb/paper/iser12.pdf', papername='Unsupervised Feature Learning for RGB-D Based Object Recognition', uncertainty=0.0, venue='ISER 2012', notes='Hierarchical sparse coding using Matching Pursuit and K-SVD')
stl10.measure(None, 62.32, 'CKN', url='http://arxiv.org/abs/1406.3332', papername='Convolutional Kernel Networks', uncertainty=0.0, venue='arXiv 2014', notes='No data augmentation.')
stl10.measure(None, 62.3, 'Discriminative Learning of Sum-Product Networks', url='http://homes.cs.washington.edu/~rcg/papers/dspn.pdf', papername='Discriminative Learning of Sum-Product Networks', uncertainty=0.0, venue='NIPS 2012', notes='')
stl10.measure(None, 61.0, 'No more meta-parameter tuning in unsupervised sparse feature learning', url='http://arxiv.org/abs/1402.5766', papername='No more meta-parameter tuning in unsupervised sparse feature learning', uncertainty=0.0, venue='arXiv 2014', notes='')
stl10.measure(None, 61.0, 'Simulated Fixations', url='http://papers.nips.cc/paper/4730-deep-learning-of-invariant-features-via-simulated-fixations-in-video', papername='Deep Learning of Invariant Features via Simulated Fixations in Video', uncertainty=0.0, venue='NIPS 2012 2012', notes='')
stl10.measure(None, 60.1, 'Receptive Fields', url='http://www.stanford.edu/~acoates/papers/coatesng_nips_2011.pdf', papername='Selecting Receptive Fields in Deep Networks ', uncertainty=0.0, venue='NIPS 2011', notes='')
stl10.measure(None, 58.7, 'Invariant Representations with Local Transformations', url='http://web.eecs.umich.edu/~honglak/icml12-invariantFeatureLearning.pdf', papername='Learning Invariant Representations with Local Transformations', uncertainty=0.0, venue='ICML 2012', notes='')
stl10.measure(None, 58.28, 'Pooling-Invariant', url='http://arxiv.org/pdf/1302.5056v1.pdf', papername='Pooling-Invariant Image Feature Learning ', uncertainty=0.0, venue='arXiv 2012', notes='1600 codes, learnt using 2x PDL')
stl10.measure(None, 56.5, 'Deep Learning of Invariant Features via Simulated Fixations in Video', url='http://ai.stanford.edu/~wzou/nips_ZouZhuNgYu12.pdf', papername='Deep Learning of Invariant Features via Simulated Fixations in Video', uncertainty=0.0, venue='NIPS 2012', notes='Trained also with video (unrelated to STL-10) obtained 61%')
# Handling 'SVHN' classification_datasets_results.html#5356484e
svhn.measure(None, 1.69, 'Tree+Max-Avg pooling', url='http://arxiv.org/abs/1509.08985', papername='Generalizing Pooling Functions in Convolutional Neural Networks: Mixed, Gated, and Tree', uncertainty=0.0, venue='AISTATS 2016', notes='Single model without data augmentation')
svhn.measure(None, 1.76, 'CMsC', url='http://arxiv.org/abs/1511.05635', papername='Competitive Multi-scale Convolution', uncertainty=0.0, venue='arXiv 2015', notes='')
svhn.measure(None, 1.77, 'RCNN-96', url='http://www.cv-foundation.org/openaccess/content_cvpr_2015/app/2B_004.pdf', papername='Recurrent Convolutional Neural Network for Object Recognition', uncertainty=0.0, venue='CVPR 2015', notes='Without data augmentation')
svhn.measure(None, 1.81, 'BNM NiN', url='http://arxiv.org/abs/1511.02583', papername='Batch-normalized Maxout Network in Network', uncertainty=0.0, venue='arXiv 2015', notes='(k=5 maxout pieces in each maxout unit).')
svhn.measure(None, 1.92, 'DSN', url='http://vcl.ucsd.edu/~sxie/2014/09/12/dsn-project/', papername='Deeply-Supervised Nets', uncertainty=0.0, venue='arXiv 2014', notes='')
svhn.measure(None, 1.92, 'MLR DNN', url='http://ieeexplore.ieee.org/xpls/abs_all.jsp?arnumber=7258343', papername='Multi-Loss Regularized Deep Neural Network', uncertainty=0.0, venue='CSVT 2015', notes='Based on NiN architecture.')
svhn.measure(None, 1.94, 'Regularization of Neural Networks using DropConnect', url='http://cs.nyu.edu/~wanli/dropc/', papername='Regularization of Neural Networks using DropConnect', uncertainty=0.0, venue='ICML 2013', notes='')
svhn.measure(None, 1.97, 'MIM', url='http://arxiv.org/abs/1508.00330', papername='On the Importance of Normalisation Layers in Deep Learning with Piecewise Linear Activation Units', uncertainty=0.08, venue='arXiv 2015', notes='')
#svhn.measure(None, 2.0, 'Estimated human performance', url='http://ufldl.stanford.edu/housenumbers/nips2011_housenumbers.pdf', papername='Estimated human performance', uncertainty=0.0, venue='NIPS 2011', notes='Based on the paper that introduced the dataset Reading Digits in Natural Images with Unsupervised Feature Learning, section 5.')
#Skipping apparent human performance (target_source) paper http://ufldl.stanford.edu/housenumbers/nips2011_housenumbers.pdf
svhn.measure(None, 2.15, 'BinaryConnect', url='http://papers.nips.cc/paper/5647-binaryconnect-training-deep-neural-networks-with-binary-weights-during-propagations.pdf', papername='BinaryConnect: Training Deep Neural Networks with binary weights during propagations', uncertainty=0.0, venue='NIPS 2015', notes='')
svhn.measure(None, 2.16, 'DCNN', url='http://openreview.net/document/0c571b22-f4b6-4d58-87e4-99d7de42a893#0c571b22-f4b6-4d58-87e4-99d7de42a893', papername='Multi-digit Number Recognition from Street View Imagery using Deep Convolutional Neural Networks', uncertainty=0.0, venue='ICLR 2014', notes='For classification of individual digits with a single network, error rate is 2.16%. For classification of the entire digit sequence (first paper doing this): error rate of 3.97%.')
svhn.measure(None, 2.35, 'NiN', url='http://openreview.net/document/9b05a3bb-3a5e-49cb-91f7-0f482af65aea#9b05a3bb-3a5e-49cb-91f7-0f482af65aea', papername='Network in Network', uncertainty=0.0, venue='ICLR 2014', notes='NIN + Dropout The code for NIN available at https://github.com/mavenlin/cuda-convnet')
svhn.measure(None, 2.38, 'ReNet', url='http://arxiv.org/abs/1505.00393', papername='ReNet: A Recurrent Neural Network Based Alternative to Convolutional Networks', uncertainty=0.0, venue='arXiv 2015', notes='')
svhn.measure(None, 2.47, 'Maxout', url='http://jmlr.org/proceedings/papers/v28/goodfellow13.pdf', papername='Maxout Networks', uncertainty=0.0, venue='ICML 2013', notes='This result was obtained using convolution but not any synthetic transformations of the training data.')
svhn.measure(None, 2.8, 'Stochastic Pooling', url='http://arxiv.org/pdf/1301.3557.pdf', papername='Stochastic Pooling for Regularization of Deep Convolutional Neural Networks', uncertainty=0.0, venue='arXiv 2013', notes='64-64-128 Stochastic Pooling')
svhn.measure(None, 3.96, 'FLSCNN', url='http://arxiv.org/abs/1503.04596', papername='Enhanced Image Classification With a Fast-Learning Shallow Convolutional Neural Network', uncertainty=0.0, venue='arXiv 2015', notes='No data augmentation')
svhn.measure(None, 4.9, u'Convolutional neural networks applied to house numbers digit classification', url='http://yann.lecun.com/exdb/publis/pdf/sermanet-icpr-12.pdf', papername='Convolutional neural networks applied to house numbers digit classification', uncertainty=0.0, venue='ICPR 2012', notes='ConvNet / MS / L4 / Padded')
# Handling 'CIFAR-100' classification_datasets_results.html#43494641522d313030
cifar100.measure(None, 75.72, 'Exponential Linear Units', url='http://arxiv.org/abs/1511.07289', papername='Fast and Accurate Deep Network Learning by Exponential Linear Units', uncertainty=0.0, venue='arXiv 2015', notes='Without data augmentation.')
cifar100.measure(None, 75.7, 'SSCNN', url='http://arxiv.org/abs/1409.6070', papername='Spatially-sparse convolutional neural networks', uncertainty=0.0, venue='arXiv 2014', notes='')
cifar100.measure(None, 73.61, 'Fractional MP', url='http://arxiv.org/abs/1412.6071', papername='Fractional Max-Pooling', uncertainty=0.0, venue='arXiv 2015', notes='Uses 12 passes at test time. Reaches 68.55% when using a single pass at test time. Uses data augmentation during training.')
cifar100.measure(None, 72.6, 'Tuned CNN', url='http://arxiv.org/abs/1502.05700', papername='Scalable Bayesian Optimization Using Deep Neural Networks', uncertainty=0.0, venue='ICML 2015', notes='')
cifar100.measure(None, 72.44, 'CMsC', url='http://arxiv.org/abs/1511.05635', papername='Competitive Multi-scale Convolution', uncertainty=0.0, venue='arXiv 2015', notes='')
cifar100.measure(None, 72.34, 'Fitnet4-LSUV', url='http://arxiv.org/abs/1511.06422', papername='All you need is a good init', uncertainty=0.0, venue='ICLR 2015', notes='Using RMSProp optimizer')
cifar100.measure(None, 71.14, 'BNM NiN', url='http://arxiv.org/abs/1511.02583', papername='Batch-normalized Maxout Network in Network', uncertainty=0.0, venue='arXiv 2015', notes='(k=5 maxout pieces in each maxout unit).')
cifar100.measure(None, 70.8, 'MIM', url='http://arxiv.org/abs/1508.00330', papername='On the Importance of Normalisation Layers in Deep Learning with Piecewise Linear Activation Units', uncertainty=0.2, venue='arXiv 2015', notes='')
cifar100.measure(None, 69.17, 'NiN+APL', url='http://arxiv.org/abs/1412.6830', papername='Learning Activation Functions to Improve Deep Neural Networks', uncertainty=0.0, venue='ICLR 2015', notes='Uses a piecewise linear activation function. 69.17% accuracy with data augmentation and 65.6% accuracy without data augmentation.')
cifar100.measure(None, 69.12, 'SWWAE', url='http://arxiv.org/abs/1506.02351', papername='Stacked What-Where Auto-encoders', uncertainty=0.0, venue='arXiv 2015', notes='')
cifar100.measure(None, 68.53, 'MLR DNN', url='http://ieeexplore.ieee.org/xpls/abs_all.jsp?arnumber=7258343', papername='Multi-Loss Regularized Deep Neural Network', uncertainty=0.0, venue='CSVT 2015', notes='With data augmentation, 65.82% without. Based on NiN architecture.')
cifar100.measure(None, 68.4, 'Spectral Representations for Convolutional Neural Networks', url='http://papers.nips.cc/paper/5649-spectral-representations-for-convolutional-neural-networks.pdf', papername='Spectral Representations for Convolutional Neural Networks', uncertainty=0.0, venue='NIPS 2015', notes='')
cifar100.measure(None, 68.25, 'RCNN-96', url='http://www.cv-foundation.org/openaccess/content_cvpr_2015/app/2B_004.pdf', papername='Recurrent Convolutional Neural Network for Object Recognition', uncertainty=0.0, venue='CVPR 2015', notes='')
cifar100.measure(None, 67.76, 'VDN', url='http://people.idsia.ch/~rupesh/very_deep_learning/', papername='Training Very Deep Networks', uncertainty=0.0, venue='NIPS 2015', notes='Best result selected on test set. 67.61% average over multiple trained models.')
cifar100.measure(None, 67.68, 'DCNN+GFE', url='http://www.isip.uni-luebeck.de/fileadmin/uploads/tx_wapublications/hertel_ijcnn_2015.pdf', papername='Deep Convolutional Neural Networks as Generic Feature Extractors', uncertainty=0.0, venue='IJCNN 2015', notes='feature extraction part of convnet is trained on imagenet (external training data), classification part is trained on cifar-100')
cifar100.measure(None, 67.63, 'Tree+Max-Avg pooling', url='http://arxiv.org/abs/1509.08985', papername='Generalizing Pooling Functions in Convolutional Neural Networks: Mixed, Gated, and Tree', uncertainty=0.0, venue='AISTATS 2016', notes='Single model without data augmentation')
cifar100.measure(None, 67.38, 'HD-CNN', url='https://sites.google.com/site/homepagezhichengyan/home/hdcnn', papername='HD-CNN: Hierarchical Deep Convolutional Neural Network for Large Scale Visual Recognition', uncertainty=0.0, venue='ICCV 2015', notes='')
cifar100.measure(None, 67.16, 'Universum Prescription', url='http://arxiv.org/abs/1511.03719', papername='Universum Prescription: Regularization using Unlabeled Data', uncertainty=0.0, venue='arXiv 2015', notes='')
cifar100.measure(None, 66.29, 'ACN', url='http://arxiv.org/pdf/1412.6806.pdf', papername='Striving for Simplicity: The All Convolutional Net', uncertainty=0.0, venue='ICLR 2014', notes='')
cifar100.measure(None, 66.22, 'Deep Networks with Internal Selective Attention through Feedback Connections', url='http://papers.nips.cc/paper/5276-deep-networks-with-internal-selective-attention-through-feedback-connections.pdf', papername='Deep Networks with Internal Selective Attention through Feedback Connections', uncertainty=0.0, venue='NIPS 2014', notes='')
cifar100.measure(None, 65.43, 'DSN', url='http://vcl.ucsd.edu/~sxie/2014/09/12/dsn-project/', papername='Deeply-Supervised Nets', uncertainty=0.0, venue='arXiv 2014', notes='Single model, without data augmentation.')
cifar100.measure(None, 64.77, 'Deep Representation Learning with Target Coding', url='http://personal.ie.cuhk.edu.hk/~ccloy/files/aaai_2015_target_coding.pdf', papername='Deep Representation Learning with Target Coding', uncertainty=0.0, venue='AAAI 2015', notes='')
cifar100.measure(None, 64.32, 'NiN', url='http://openreview.net/document/9b05a3bb-3a5e-49cb-91f7-0f482af65aea#9b05a3bb-3a5e-49cb-91f7-0f482af65aea', papername='Network in Network', uncertainty=0.0, venue='ICLR 2014', notes='NIN + Dropout The code for NIN available at https://github.com/mavenlin/cuda-convnet')
cifar100.measure(None, 63.15, 'Tree Priors', url='http://www.cs.toronto.edu/~nitish/treebasedpriors.pdf', papername='Discriminative Transfer Learning with Tree-based Priors', uncertainty=0.0, venue='NIPS 2013', notes=u'The baseline Convnet + max pooling + dropout reaches 62.80% (without any tree prior).')
cifar100.measure(None, 61.86, 'DNN+Probabilistic Maxout', url='http://openreview.net/document/28d9c3ab-fe88-4836-b898-403d207a037c#28d9c3ab-fe88-4836-b898-403d207a037c', papername='Improving Deep Neural Networks with Probabilistic Maxout Units', uncertainty=0.0, venue='ICLR 2014', notes='')
cifar100.measure(None, 61.43, 'Maxout Networks', url='http://jmlr.org/proceedings/papers/v28/goodfellow13.pdf', papername='Maxout Networks', uncertainty=0.0, venue='ICML 2013', notes='Uses convolution. Does not use dataset agumentation.')
cifar100.measure(None, 60.8, 'Stable and Efficient Representation Learning with Nonnegativity Constraints ', url='http://jmlr.org/proceedings/papers/v32/line14.pdf', papername='Stable and Efficient Representation Learning with Nonnegativity Constraints ', uncertainty=0.0, venue='ICML 2014', notes='3-layers + multi-dict. 7 with 3-layers only. 3 with 1-layers only.')
cifar100.measure(None, 59.75, 'RReLU', url='http://arxiv.org/pdf/1505.00853.pdf', papername='Empirical Evaluation of Rectified Activations in Convolution Network', uncertainty=0.0, venue='ICML workshop 2015', notes='Using Randomized Leaky ReLU')
cifar100.measure(None, 57.49, 'Stochastic Pooling', url='http://arxiv.org/pdf/1301.3557.pdf', papername='Stochastic Pooling for Regularization of Deep Convolutional Neural Networks', uncertainty=0.0, venue='arXiv 2013', notes='')
cifar100.measure(None, 56.29, 'Smooth Pooling Regions', url='http://www.d2.mpi-inf.mpg.de/content/learning-smooth-pooling-regions-visual-recognition', papername='Smooth Pooling Regions', uncertainty=0.0, venue='BMVC 2013', notes='No data augmentation.')
cifar100.measure(None, 54.23, 'Receptive Field Learning', url='http://www.eecs.berkeley.edu/~jiayq/assets/pdf/cvpr12_pooling.pdf', papername='Beyond Spatial Pyramids: Receptive Field Learning for Pooled Image Features', uncertainty=0.0, venue='CVPR 2012', notes='')
# Handling 'CIFAR-10' classification_datasets_results.html#43494641522d3130
cifar10.measure(None, 96.53, 'Fractional MP', url='http://arxiv.org/abs/1412.6071', papername='Fractional Max-Pooling', uncertainty=0.0, venue='arXiv 2015', notes='Uses 100 passes at test time. Reaches 95.5% when using a single pass at test time, and 96.33% when using 12 passes.. Uses data augmentation during training.')
cifar10.measure(None, 95.59, 'ACN', url='http://arxiv.org/pdf/1412.6806.pdf', papername='Striving for Simplicity: The All Convolutional Net', uncertainty=0.0, venue='ICLR 2015', notes='92% without data augmentation, 92.75% with small data augmentation, 95.59% when using agressive data augmentation and larger network.')
cifar10.measure(None, 94.16, 'Fitnet4-LSUV', url='http://arxiv.org/abs/1511.06422', papername='All you need is a good init', uncertainty=0.0, venue='ICLR 2016', notes='Only mirroring and random shifts, no extreme data augmentation. Uses thin deep residual net with maxout activations.')
#cifar10.measure(None, 94.0, 'Lessons learned from manually classifying CIFAR-10', url='http://karpathy.github.io/2011/04/27/manually-classifying-cifar10/', papername='Lessons learned from manually classifying CIFAR-10', uncertainty=0.0, venue='unpublished 2011', notes='Rough estimate from a single individual, over 400 training images (~1% of training data).')
#Skipping apparent human performance (target_source) paper http://karpathy.github.io/2011/04/27/manually-classifying-cifar10/
cifar10.measure(None, 93.95, 'Tree+Max-Avg pooling', url='http://arxiv.org/abs/1509.08985', papername='Generalizing Pooling Functions in Convolutional Neural Networks: Mixed, Gated, and Tree', uncertainty=0.0, venue='AISTATS 2016', notes='Single model with data augmentation, 92.38% without.')
cifar10.measure(None, 93.72, 'SSCNN', url='http://arxiv.org/abs/1409.6070', papername='Spatially-sparse convolutional neural networks', uncertainty=0.0, venue='arXiv 2014', notes='')
cifar10.measure(None, 93.63, 'Tuned CNN', url='http://arxiv.org/abs/1502.05700', papername='Scalable Bayesian Optimization Using Deep Neural Networks', uncertainty=0.0, venue='ICML 2015', notes='')
cifar10.measure(None, 93.57, 'DRL', url='http://arxiv.org/abs/1512.03385', papername='Deep Residual Learning for Image Recognition', uncertainty=0.0, venue='arXiv 2015', notes='Best performance reached with 110 layers. Using 1202 layers leads to 92.07%, 56 layers lead to 93.03%.')
cifar10.measure(None, 93.45, 'Exponential Linear Units', url='http://arxiv.org/abs/1511.07289', papername='Fast and Accurate Deep Network Learning by Exponential Linear Units', uncertainty=0.0, venue='arXiv 2015', notes='Without data augmentation.')
cifar10.measure(None, 93.34, 'Universum Prescription', url='http://arxiv.org/abs/1511.03719', papername='Universum Prescription: Regularization using Unlabeled Data', uncertainty=0.0, venue='arXiv 2015', notes='')
cifar10.measure(None, 93.25, 'BNM NiN', url='http://arxiv.org/abs/1511.02583', papername='Batch-normalized Maxout Network in Network', uncertainty=0.0, venue='arXiv 2015', notes='(k=5 maxout pieces in each maxout unit). Reaches 92.15% without data augmentation.')
cifar10.measure(None, 93.13, 'CMsC', url='http://arxiv.org/abs/1511.05635', papername='Competitive Multi-scale Convolution', uncertainty=0.0, venue='arXiv 2015', notes='')
cifar10.measure(None, 92.91, 'RCNN-96', url='http://www.cv-foundation.org/openaccess/content_cvpr_2015/app/2B_004.pdf', papername='Recurrent Convolutional Neural Network for Object Recognition', uncertainty=0.0, venue='CVPR 2015', notes='Reaches 91.31% without data augmentation.')
cifar10.measure(None, 92.49, 'NiN+APL', url='http://arxiv.org/abs/1412.6830', papername='Learning Activation Functions to Improve Deep Neural Networks', uncertainty=0.0, venue='ICLR 2015', notes='Uses an adaptive piecewise linear activation function. 92.49% accuracy with data augmentation and 90.41% accuracy without data augmentation.')
cifar10.measure(None, 92.45, 'cifar.torch', url='http://torch.ch/blog/2015/07/30/cifar.html', papername='cifar.torch', uncertainty=0.0, venue='unpublished 2015', notes='Code available at https://github.com/szagoruyko/cifar.torch')
cifar10.measure(None, 92.4, 'VDN', url='http://people.idsia.ch/~rupesh/very_deep_learning/', papername='Training Very Deep Networks', uncertainty=0.0, venue='NIPS 2015', notes='Best result selected on test set. 92.31% average over multiple trained models.')
cifar10.measure(None, 92.23, 'SWWAE', url='http://arxiv.org/abs/1506.02351', papername='Stacked What-Where Auto-encoders', uncertainty=0.0, venue='arXiv 2015', notes='')
cifar10.measure(None, 91.88, 'MLR DNN', url='http://ieeexplore.ieee.org/xpls/abs_all.jsp?arnumber=7258343', papername='Multi-Loss Regularized Deep Neural Network', uncertainty=0.0, venue='CSVT 2015', notes='With data augmentation, 90.45% without. Based on NiN architecture.')
cifar10.measure(None, 91.78, 'DSN', url='http://vcl.ucsd.edu/~sxie/2014/09/12/dsn-project/', papername='Deeply-Supervised Nets', uncertainty=0.0, venue='arXiv 2014', notes='Single model, with data augmentation: 91.78%. Without data augmentation: 90.22%.')
cifar10.measure(None, 91.73, 'BinaryConnect', url='http://papers.nips.cc/paper/5647-binaryconnect-training-deep-neural-networks-with-binary-weights-during-propagations.pdf', papername='BinaryConnect: Training Deep Neural Networks with binary weights during propagations', uncertainty=0.0, venue='NIPS 2015', notes='These results were obtained without using any data-augmentation.')
cifar10.measure(None, 91.48, 'MIM', url='http://arxiv.org/abs/1508.00330', papername='On the Importance of Normalisation Layers in Deep Learning with Piecewise Linear Activation Units', uncertainty=0.2, venue='arXiv 2015', notes='')
cifar10.measure(None, 91.4, 'Spectral Representations for Convolutional Neural Networks', url='http://papers.nips.cc/paper/5649-spectral-representations-for-convolutional-neural-networks.pdf', papername='Spectral Representations for Convolutional Neural Networks', uncertainty=0.0, venue='NIPS 2015', notes='')
cifar10.measure(None, 91.2, 'NiN', url='http://openreview.net/document/9b05a3bb-3a5e-49cb-91f7-0f482af65aea#9b05a3bb-3a5e-49cb-91f7-0f482af65aea', papername='Network In Network', uncertainty=0.0, venue='ICLR 2014', notes='The code for NIN available at https://github.com/mavenlin/cuda-convnet NIN + Dropout 89.6% NIN + Dropout + Data Augmentation 91.2%')
cifar10.measure(None, 91.19, 'ELC', url='http://aad.informatik.uni-freiburg.de/papers/15-IJCAI-Extrapolation_of_Learning_Curves.pdf', papername='Speeding up Automatic Hyperparameter Optimization of Deep Neural Networks by Extrapolation of Learning Curves', uncertainty=0.0, venue='IJCAI 2015', notes=u'Based on the "call convolutional" architecture. which reaches 90.92% by itself.')
cifar10.measure(None, 90.78, 'Deep Networks with Internal Selective Attention through Feedback Connections', url='http://papers.nips.cc/paper/5276-deep-networks-with-internal-selective-attention-through-feedback-connections.pdf', papername='Deep Networks with Internal Selective Attention through Feedback Connections', uncertainty=0.0, venue='NIPS 2014', notes='No data augmentation')
cifar10.measure(None, 90.68, 'DropConnect', url='http://cs.nyu.edu/~wanli/dropc/', papername='Regularization of Neural Networks using DropConnect', uncertainty=0.0, venue='ICML 2013', notes='')
cifar10.measure(None, 90.65, 'Maxout Networks', url='http://jmlr.org/proceedings/papers/v28/goodfellow13.pdf', papername='Maxout Networks', uncertainty=0.0, venue='ICML 2013', notes='This result was obtained using both convolution and synthetic translations / horizontal reflections of the training data. Reaches 88.32% when using convolution, but without any synthetic transformations of the training data.')
cifar10.measure(None, 90.61, 'DNN+Probabilistic Maxout', url='http://openreview.net/document/28d9c3ab-fe88-4836-b898-403d207a037c#28d9c3ab-fe88-4836-b898-403d207a037c', papername='Improving Deep Neural Networks with Probabilistic Maxout Units', uncertainty=0.0, venue='ICLR 2014', notes='65% without data augmentation. 61% when using data augmentation.')
cifar10.measure(None, 90.5, 'GP EI', url='http://papers.nips.cc/paper/4522-practical-bayesian-optimization-of-machine-learning-algorithms.pdf', papername='Practical Bayesian Optimization of Machine Learning Algorithms ', uncertainty=0.0, venue='NIPS 2012', notes=u'Reaches 85.02% without data augmentation. With data augmented with horizontal reflections and translations, 90.5% accuracy on test set is achieved.')
cifar10.measure(None, 89.67, 'APAC', url='http://arxiv.org/abs/1505.03229', papername='APAC: Augmented PAttern Classification with Neural Networks', uncertainty=0.0, venue='arXiv 2015', notes='')
cifar10.measure(None, 89.14, 'DCNN+GFE', url='http://www.isip.uni-luebeck.de/fileadmin/uploads/tx_wapublications/hertel_ijcnn_2015.pdf', papername='Deep Convolutional Neural Networks as Generic Feature Extractors', uncertainty=0.0, venue='IJCNN 2015', notes='feature extraction part of convnet is trained on imagenet (external training data), classification part is trained on cifar-10')
cifar10.measure(None, 89.0, 'DCNN', url='http://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks', papername='ImageNet Classification with Deep Convolutional Neural Networks', uncertainty=0.0, venue='NIPS 2012', notes='87% error on the unaugmented data.')
cifar10.measure(None, 88.8, 'RReLU', url='http://arxiv.org/pdf/1505.00853.pdf', papername='Empirical Evaluation of Rectified Activations in Convolution Network', uncertainty=0.0, venue='ICML workshop 2015', notes='Using Randomized Leaky ReLU')
cifar10.measure(None, 88.79, 'MCDNN', url='http://www.idsia.ch/~ciresan/data/cvpr2012.pdf', papername='Multi-Column Deep Neural Networks for Image Classification ', uncertainty=0.0, venue='CVPR 2012', notes='Supplemental material, Technical Report')
cifar10.measure(None, 87.65, 'ReNet', url='http://arxiv.org/abs/1505.00393', papername='ReNet: A Recurrent Neural Network Based Alternative to Convolutional Networks', uncertainty=0.0, venue='arXiv 2015', notes='')
cifar10.measure(None, 86.7, 'An Analysis of Unsupervised Pre-training in Light of Recent Advances', url='http://arxiv.org/abs/1412.6597', papername='An Analysis of Unsupervised Pre-training in Light of Recent Advances', uncertainty=0.0, venue='ICLR 2015', notes='Unsupervised pre-training, with supervised fine-tuning. Uses dropout and data-augmentation.')
cifar10.measure(None, 84.87, 'Stochastic Pooling', url='http://arxiv.org/pdf/1301.3557.pdf', papername='Stochastic Pooling for Regularization of Deep Convolutional Neural Networks', uncertainty=0.0, venue='arXiv 2013', notes='')
cifar10.measure(None, 84.4, 'Improving neural networks by preventing co-adaptation of feature detectors', url='http://arxiv.org/pdf/1207.0580.pdf', papername='Improving neural networks by preventing co-adaptation of feature detectors', uncertainty=0.0, venue='arXiv 2012', notes=u'So called "dropout" method.')
cifar10.measure(None, 83.96, 'Discriminative Learning of Sum-Product Networks', url='http://papers.nips.cc/paper/4516-discriminative-learning-of-sum-product-networks', papername='Discriminative Learning of Sum-Product Networks', uncertainty=0.0, venue='NIPS 2012', notes='')
cifar10.measure(None, 82.9, 'Nonnegativity Constraints ', url='http://jmlr.org/proceedings/papers/v32/line14.pdf', papername='Stable and Efficient Representation Learning with Nonnegativity Constraints ', uncertainty=0.0, venue='ICML 2014', notes='Full data, 3-layers + multi-dict. 4 with 3-layers only. 0 with 1-layers only.')
cifar10.measure(None, 82.2, 'Local Transformations', url='http://icml.cc/2012/papers/659.pdf', papername='Learning Invariant Representations with Local Transformations', uncertainty=0.0, venue='ICML 2012', notes='K= 4,000')
cifar10.measure(None, 82.18, 'CKN', url='http://arxiv.org/abs/1406.3332', papername='Convolutional Kernel Networks', uncertainty=0.0, venue='arXiv 2014', notes='No data augmentation.')
cifar10.measure(None, 82.0, 'Discriminative Unsupervised Feature Learning with Convolutional Neural Networks', url='http://papers.nips.cc/paper/5548-discriminative-unsupervised-feature-learning-with-convolutional-neural-networks.pdf', papername='Discriminative Unsupervised Feature Learning with Convolutional Neural Networks', uncertainty=0.0, venue='NIPS 2014', notes='Unsupervised feature learning + linear SVM')
cifar10.measure(None, 80.02, 'Smooth Pooling Regions', url='http://www.d2.mpi-inf.mpg.de/content/learning-smooth-pooling-regions-visual-recognition', papername='Learning Smooth Pooling Regions for Visual Recognition', uncertainty=0.0, venue='BMVC 2013', notes='')
cifar10.measure(None, 80.0, 'Hierarchical Kernel Descriptors', url='http://research.cs.washington.edu/istc/lfb/paper/cvpr11.pdf', papername='Object Recognition with Hierarchical Kernel Descriptors', uncertainty=0.0, venue='CVPR 2011', notes='')
cifar10.measure(None, 79.7, 'Learning with Recursive Perceptual Representations', url='http://papers.nips.cc/paper/4747-learning-with-recursive-perceptual-representations', papername='Learning with Recursive Perceptual Representations', uncertainty=0.0, venue='NIPS 2012', notes='Code size 1600.')
cifar10.measure(None, 79.6, 'An Analysis of Single-Layer Networks in Unsupervised Feature Learning ', url='http://www.stanford.edu/~acoates/papers/coatesleeng_aistats_2011.pdf', papername='An Analysis of Single-Layer Networks in Unsupervised Feature Learning ', uncertainty=0.0, venue='AISTATS 2011', notes='6% obtained using K-means over whitened patches, with triangle encoding and 4000 features (clusters).')
cifar10.measure(None, 78.67, 'PCANet', url='http://arxiv.org/abs/1404.3606', papername='PCANet: A Simple Deep Learning Baseline for Image Classification?', uncertainty=0.0, venue='arXiv 2014', notes='No data augmentation. Multiple feature scales combined. 77.14% when using only a single scale.')
cifar10.measure(None, 75.86, 'FLSCNN', url='http://arxiv.org/abs/1503.04596', papername='Enhanced Image Classification With a Fast-Learning Shallow Convolutional Neural Network', uncertainty=0.0, venue='arXiv 2015', notes='No data augmentation')
# Handling 'MNIST' classification_datasets_results.html#4d4e495354
mnist.measure(None, 0.21, 'DropConnect', url='http://cs.nyu.edu/~wanli/dropc/', papername='Regularization of Neural Networks using DropConnect', uncertainty=0.0, venue='ICML 2013', notes='')
mnist.measure(None, 0.23, 'MCDNN', url='http://www.idsia.ch/~ciresan/data/cvpr2012.pdf', papername=u'Multi-column Deep Neural Networks for Image Classification ', uncertainty=0.0, venue='CVPR 2012', notes='')
mnist.measure(None, 0.23, 'APAC', url='http://arxiv.org/abs/1505.03229', papername='APAC: Augmented PAttern Classification with Neural Networks', uncertainty=0.0, venue='arXiv 2015', notes='')
mnist.measure(None, 0.24, 'BNM NiN', url='http://arxiv.org/abs/1511.02583', papername='Batch-normalized Maxout Network in Network', uncertainty=0.0, venue='arXiv 2015', notes='(k=5 maxout pieces in each maxout unit).')
mnist.measure(None, 0.29, 'Tree+Max-Avg pooling', url='http://arxiv.org/abs/1509.08985', papername='Generalizing Pooling Functions in Convolutional Neural Networks: Mixed, Gated, and Tree', uncertainty=0.0, venue='AISTATS 2016', notes='Single model without data augmentation')
mnist.measure(None, 0.31, 'RCNN-96', url='http://www.cv-foundation.org/openaccess/content_cvpr_2015/app/2B_004.pdf', papername='Recurrent Convolutional Neural Network for Object Recognition', uncertainty=0.0, venue='CVPR 2015', notes='')
mnist.measure(None, 0.35, 'MIM', url='http://arxiv.org/abs/1508.00330', papername='On the Importance of Normalisation Layers in Deep Learning with Piecewise Linear Activation Units', uncertainty=0.03, venue='arXiv 2015', notes='')
mnist.measure(None, 0.32, 'Fractional MP', url='http://arxiv.org/abs/1412.6071', papername='Fractional Max-Pooling', uncertainty=0.0, venue='arXiv 2015', notes='Uses 12 passes at test time. Reaches 0.5% when using a single pass at test time.')
mnist.measure(None, 0.33, 'CMsC', url='http://arxiv.org/abs/1511.05635', papername='Competitive Multi-scale Convolution', uncertainty=0.0, venue='arXiv 2015', notes='')
mnist.measure(None, 0.35, 'DBSNN', url='http://arxiv.org/pdf/1003.0358.pdf', papername='Deep Big Simple Neural Nets Excel on Handwritten Digit Recognition', uncertainty=0.0, venue='Neural Computation 2010', notes='6-layer NN 784-2500-2000-1500-1000-500-10 (on GPU), uses elastic distortions')
mnist.measure(None, 0.35, 'C-SVDDNet', url='http://arxiv.org/abs/1412.7259', papername='C-SVDDNet: An Effective Single-Layer Network for Unsupervised Feature Learning', uncertainty=0.0, venue='arXiv 2014', notes='')
mnist.measure(None, 0.37, 'FLSCNN', url='http://arxiv.org/abs/1503.04596', papername='Enhanced Image Classification With a Fast-Learning Shallow Convolutional Neural Network', uncertainty=0.0, venue='arXiv 2015', notes='No data augmentation')
mnist.measure(None, 0.39, 'Energy-Based Sparse Represenation', url='http://papers.nips.cc/paper/3112-efficient-learning-of-sparse-representations-with-an-energy-based-model', papername=u'Efficient Learning of Sparse Representations with an Energy-Based Model', uncertainty=0.0, venue='NIPS 2006', notes='Large conv. net, unsup pretraining, uses elastic distortions')
mnist.measure(None, 0.39, 'CKN', url='http://arxiv.org/abs/1406.3332', papername='Convolutional Kernel Networks', uncertainty=0.0, venue='arXiv 2014', notes='No data augmentation.')
mnist.measure(None, 0.39, 'DSN', url='http://vcl.ucsd.edu/~sxie/2014/09/12/dsn-project/', papername='Deeply-Supervised Nets', uncertainty=0.0, venue='arXiv 2014', notes='')
mnist.measure(None, 0.4, 'Best Practices for Convolutional Neural Networks Applied to Visual Document Analysis', url='http://citeseerx.ist.psu.edu/viewdoc/download;jsessionid=D1C7D701BD39935473808DA5A93426C5?doi=10.1.1.160.8494&rep=rep1&type=pdf', papername='Best Practices for Convolutional Neural Networks Applied to Visual Document Analysis', uncertainty=0.0, venue='Document Analysis and Recognition 2003', notes='')
mnist.measure(None, 0.4, 'HOPE', url='http://arxiv.org/pdf/1502.00702.pdf', papername='Hybrid Orthogonal Projection and Estimation (HOPE): A New Framework to Probe and Learn Neural Networks', uncertainty=0.0, venue=' arXiv 2015', notes='')
mnist.measure(None, 0.42, 'MLR DNN', url='http://ieeexplore.ieee.org/xpls/abs_all.jsp?arnumber=7258343', papername='Multi-Loss Regularized Deep Neural Network', uncertainty=0.0, venue='CSVT 2015', notes='Based on NiN architecture.')
mnist.measure(None, 0.45, 'Maxout Networks', url='http://jmlr.org/proceedings/papers/v28/goodfellow13.pdf', papername='Maxout Networks', uncertainty=0.0, venue='ICML 2013', notes='Uses convolution. Does not use dataset augmentation.')
mnist.measure(None, 0.45, 'VDN', url='http://people.idsia.ch/~rupesh/very_deep_learning/', papername='Training Very Deep Networks', uncertainty=0.0, venue='NIPS 2015', notes='Best result selected on test set. 0.46% average over multiple trained models.')
mnist.measure(None, 0.45, 'ReNet', url='http://arxiv.org/abs/1505.00393', papername='ReNet: A Recurrent Neural Network Based Alternative to Convolutional Networks', uncertainty=0.0, venue='arXiv 2015', notes='')
mnist.measure(None, 0.46, 'DCNN+GFE', url='http://www.isip.uni-luebeck.de/fileadmin/uploads/tx_wapublications/hertel_ijcnn_2015.pdf', papername='Deep Convolutional Neural Networks as Generic Feature Extractors', uncertainty=0.0, venue='IJCNN 2015', notes='feature extraction part of convnet is trained on imagenet (external training data), classification part is trained on cifar-10')
mnist.measure(None, 0.47, 'NiN', url='http://openreview.net/document/9b05a3bb-3a5e-49cb-91f7-0f482af65aea#9b05a3bb-3a5e-49cb-91f7-0f482af65aea', papername='Network in Network', uncertainty=0.0, venue='ICLR 2014', notes='NIN + Dropout The code for NIN available at https://github.com/mavenlin/cuda-convnet')

# skip broken URL
#mnist.measure(None, 0.52, 'COSFIRE', url='http://iwi.eldoc.ub.rug.nl/FILES/root/2013/IEEETPAMIAzzopardi/2013IEEETPAMIAzzopardi.pdf', papername='Trainable COSFIRE filters for keypoint detection and pattern recognition', uncertainty=0.0, venue='PAMI 2013', notes='Source code available.')
mnist.measure(None, 0.53, 'The Best Multi-Stage Architecture', url='http://yann.lecun.com/exdb/publis/pdf/jarrett-iccv-09.pdf', papername='What is the Best Multi-Stage Architecture for Object Recognition?', uncertainty=0.0, venue='ICCV 2009', notes='Large conv. net, unsup pretraining, no distortions')
mnist.measure(None, 0.54, 'Deformation Models', url='http://www.keysers.net/daniel/files/Keysers--Deformation-Models--TPAMI2007.pdf', papername='Deformation Models for Image Recognition', uncertainty=0.0, venue='PAMI 2007', notes='K-NN with non-linear deformation (IDM) (Preprocessing: shiftable edges)')
mnist.measure(None, 0.54, 'Trainable feature extractor', url='http://hal.inria.fr/docs/00/05/75/61/PDF/LauerSuenBlochPR.pdf', papername='A trainable feature extractor for handwritten digit recognition', uncertainty=0.0, venue='\n                 Journal\n                Pattern Recognition 2007\n              ', notes='Trainable feature extractor + SVMs, uses affine distortions')
mnist.measure(None, 0.56, 'ISVM', url='http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.88.9924&rep=rep1&type=pdf', papername='Training Invariant Support Vector Machines', uncertainty=0.0, venue='Machine Learning 2002', notes='Virtual SVM, deg-9 poly, 2-pixel jittered (Preprocessing: deskewing)')
mnist.measure(None, 0.59, 'Sparse Coding', url='http://www.inb.uni-luebeck.de/publikationen/pdfs/LaBaMa08c.pdf', papername='Simple Methods for High-Performance Digit Recognition Based on Sparse Coding', uncertainty=0.0, venue='TNN 2008', notes='Unsupervised sparse features + SVM, no distortions')
mnist.measure(None, 0.62, 'invariant feature hierarchies', url='http://yann.lecun.com/exdb/publis/pdf/ranzato-cvpr-07.pdf', papername='Unsupervised learning of invariant feature hierarchies with applications to object recognition', uncertainty=0.0, venue='CVPR 2007', notes='Large conv. net, unsup features, no distortions')
mnist.measure(None, 0.62, 'PCANet', url='http://arxiv.org/abs/1404.3606', papername='PCANet: A Simple Deep Learning Baseline for Image Classification?', uncertainty=0.0, venue='arXiv 2014', notes='No data augmentation.')
mnist.measure(None, 0.63, 'Shape contexts', url='http://citeseerx.ist.psu.edu/viewdoc/download;jsessionid=B2AAC2BC3824F19757CAC66986D5F3FF?doi=10.1.1.18.8852&rep=rep1&type=pdf', papername='Shape matching and object recognition using shape contexts', uncertainty=0.0, venue='PAMI 2002', notes='K-NN, shape context matching (preprocessing: shape context feature extraction)')
mnist.measure(None, 0.64, 'Receptive Field Learning', url='http://www.icsi.berkeley.edu/pubs/vision/beyondspatial12.pdf', papername='Beyond Spatial Pyramids: Receptive Field Learning for Pooled Image Features', uncertainty=0.0, venue='CVPR 2012', notes='')
mnist.measure(None, 0.68, 'CNN+Gabor Filters', url='http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.133.6559&rep=rep1&type=pdf', papername='Handwritten Digit Recognition using Convolutional Neural Networks and Gabor Filters', uncertainty=0.0, venue='ICCI 2003', notes='')
mnist.measure(None, 0.69, 'On Optimization Methods for Deep Learning', url='http://ai.stanford.edu/~quocle/LeNgiCoaLahProNg11.pdf', papername='On Optimization Methods for Deep Learning', uncertainty=0.0, venue='ICML 2011', notes='')
mnist.measure(None, 0.71, 'Deep Fried Convnets', url='http://www.cv-foundation.org/openaccess/content_iccv_2015/papers/Yang_Deep_Fried_Convnets_ICCV_2015_paper.pdf', papername='Deep Fried Convnets', uncertainty=0.0, venue='ICCV 2015', notes='Uses about 10x fewer parameters than the reference model, which reaches 0.87%.')
mnist.measure(None, 0.75, 'Sparse Activity and Sparse Connectivity in Supervised Learning', url='http://jmlr.org/papers/v14/thom13a.html', papername='Sparse Activity and Sparse Connectivity in Supervised Learning', uncertainty=0.0, venue='JMLR 2013', notes='')
mnist.measure(None, 0.78, 'Explaining and Harnessing Adversarial Examples', url='http://arxiv.org/abs/1412.6572', papername='Explaining and Harnessing Adversarial Examples', uncertainty=0.0, venue='ICLR 2015', notes='permutation invariant network used')
mnist.measure(None, 0.82, 'CDBN', url=None, papername='Convolutional Deep Belief Networks for Scalable Unsupervised Learning of Hierarchical Representations', uncertainty=0.0, venue='ICML 2009', notes='')
mnist.measure(None, 0.84, 'Supervised Translation-Invariant Sparse Coding', url='http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.206.339&rep=rep1&type=pdf', papername='Supervised Translation-Invariant Sparse Coding', uncertainty=0.0, venue='CVPR 2010', notes='Uses sparse coding + svm.')
mnist.measure(None, 0.94, 'Large-Margin kNN', url=None, papername='Large-Margin kNN Classification using a Deep Encoder Network', uncertainty=0.0, venue=' 2009', notes='')
mnist.measure(None, 0.95, 'Deep Boltzmann Machines', url='http://www.utstat.toronto.edu/~rsalakhu/papers/dbm.pdf', papername='Deep Boltzmann Machines', uncertainty=0.0, venue='AISTATS 2009', notes='')
mnist.measure(None, 1.01, 'BinaryConnect', url='http://papers.nips.cc/paper/5647-binaryconnect-training-deep-neural-networks-with-binary-weights-during-propagations.pdf', papername='BinaryConnect: Training Deep Neural Networks with binary weights during propagations', uncertainty=0.0, venue='NIPS 2015', notes='Using 50% dropout')
mnist.measure(None, 1.1, 'StrongNet', url='http://www.alglib.net/articles/tr-20140813-strongnet.pdf', papername='StrongNet: mostly unsupervised image recognition with strong neurons', uncertainty=0.0, venue='technical report on ALGLIB website 2014', notes=u'StrongNet is a neural design which uses two innovations: (a) strong neurons - highly nonlinear neurons with multiple outputs and (b) mostly unsupervised architecture  backpropagation-free design with all layers except for the last one being trained in a completely unsupervised setting.')
mnist.measure(None, 1.12, 'DBN', url=None, papername='CS81: Learning words with Deep Belief Networks', uncertainty=0.0, venue=' 2008', notes='')
mnist.measure(None, 1.19, 'CNN', url=None, papername='Convolutional Neural Networks', uncertainty=0.0, venue=' 2003', notes=u'The ConvNN is based on the paper "Best Practices for Convolutional Neural Networks Applied to Visual Document Analysis".')
mnist.measure(None, 1.2, 'Reducing the dimensionality of data with neural networks', url=None, papername='Reducing the dimensionality of data with neural networks', uncertainty=0.0, venue=' 2006', notes='')
mnist.measure(None, 1.4, 'Convolutional Clustering', url='http://arxiv.org/abs/1511.06241', papername='Convolutional Clustering for Unsupervised Learning', uncertainty=0.0, venue='arXiv 2015', notes='2 layers + multi dict.')
mnist.measure(None, 1.5, 'Deep learning via semi-supervised embedding', url=None, papername='Deep learning via semi-supervised embedding', uncertainty=0.0, venue=' 2008', notes='')
# This paper has reasonable results on other things, but one so bad on MNIST that it messes up the graph. So we exclude it:
# mnist.measure(None, 14.53, 'Deep Representation Learning with Target Coding', url='http://personal.ie.cuhk.edu.hk/~ccloy/files/aaai_2015_target_coding.pdf', papername='Deep Representation Learning with Target Coding', uncertainty=0.0, venue='AAAI 2015', notes='')

# Handling 'MSRC-21' semantic_labeling_datasets_results.html#4d5352432d3231
msrc21_pc.measure(None, 80.9, 'Large FC CRF', url='http://ai2-s2-pdfs.s3.amazonaws.com/daba/eb9185990f65f807c95ff4d09057c2bf1cf0.pdf', papername='Large-Scale Semantic Co-Labeling of Image Sets', uncertainty=0.0, venue='WACV 2014', notes='')
msrc21_pc.measure(None, 80.0, 'Harmony Potentials', url='http://link.springer.com/article/10.1007%2Fs11263-011-0449-8', papername='Harmony Potentials - Fusing Local and Global Scale for Semantic Image Segmentation', uncertainty=0.0, venue='IJCV 2012', notes='per-class % / per-pixel %')
msrc21_pc.measure(None, 79.0, 'Describing the Scene as a Whole: Joint Object Detection, Scene Classification and Semantic Segmentation', url='http://ttic.uchicago.edu/~rurtasun/publications/yao_et_al_cvpr12.pdf', papername='Describing the Scene as a Whole: Joint Object Detection, Scene Classification and Semantic Segmentation', uncertainty=0.0, venue='CVPR 2012', notes='')
msrc21_pc.measure(None, 78.2, 'MPP', url='http://mediatum.ub.tum.de/doc/1175516/1175516.pdf', papername='Morphological Proximity Priors: Spatial Relationships for Semantic Segmentation', uncertainty=0.0, venue='TUM-I1222 2013', notes='')
msrc21_pc.measure(None, 78.0, 'FC CRF', url='http://graphics.stanford.edu/projects/densecrf/densecrf.pdf', papername='Efficient Inference in Fully Connected CRFs with Gaussian Edge Potentials', uncertainty=0.0, venue='NIPS 2011', notes='Strong unary used provides 76.6% / 84.0%')
msrc21_pc.measure(None, 77.0, 'HCRF+CO', url='http://research.microsoft.com/en-us/um/people/pkohli/papers/lrkt_eccv2010.pdf', papername='Graph Cut based Inference with Co-occurrence Statistics', uncertainty=0.0, venue='ECCV 2010', notes='')
msrc21_pc.measure(None, 77.0, 'Are Spatial and Global Constraints Really Necessary for Segmentation?', url='http://infoscience.epfl.ch/record/169178/files/lucchi_ICCV11.pdf', papername='Are Spatial and Global Constraints Really Necessary for Segmentation?', uncertainty=0.0, venue='ICCV 2011', notes='Several variants are examined, no single method attains the overall best results, i.e. both best per-class and per-pixel averages simultaneously. Indicated result corresponds to the method that we best on the average (per-class + per-pixel / 2). Experiment data available.')
msrc21_pc.measure(None, 76.0, 'Kernelized SSVM/CRF', url='https://infoscience.epfl.ch/record/180188/files/top.pdf', papername='Structured Image Segmentation using Kernelized Features', uncertainty=0.0, venue='ECCV 2012', notes='70 % / 73 % when using only local features (not considering global features)')
msrc21_pc.measure(None, 72.8, 'PMG', url='http://users.cecs.anu.edu.au/~sgould/papers/eccv12-patchGraph.pdf', papername='PatchMatchGraph: Building a Graph of Dense Patch Correspondences for Label Transfer', uncertainty=0.0, venue='ECCV 2012', notes='8% / 63.3% raw PatchMatchGraph accuracy, 72.8% / 79.0% when using Boosted CRF. Code available.')
msrc21_pc.measure(None, 69.0, 'Auto-Context', url='http://pages.ucsd.edu/~ztu/publication/pami_autocontext.pdf', papername='Auto-Context and Its Application to High-Level Vision Tasks and 3D Brain Image Segmentation', uncertainty=0.0, venue='PAMI 2010', notes='')
msrc21_pc.measure(None, 67.0, 'STF', url='http://mi.eng.cam.ac.uk/~cipolla/publications/inproceedings/2008-CVPR-semantic-texton-forests.pdf', papername='Semantic Texton Forests for Image Categorization and Segmentation', uncertainty=0.0, venue='CVPR 2008', notes='')
msrc21_pc.measure(None, 57.0, 'TextonBoost', url='http://research.microsoft.com/pubs/117885/ijcv07a.pdf', papername='TextonBoost for Image Understanding', uncertainty=0.0, venue='IJCV 2009', notes='?? / 69.6 % (per-class / per-pixel) the unaries alone (no CRF on top)')
# Handling 'MSRC-21' semantic_labeling_datasets_results.html#4d5352432d3231
msrc21_pp.measure(None, 86.8, 'Large FC CRF', url='http://ai2-s2-pdfs.s3.amazonaws.com/daba/eb9185990f65f807c95ff4d09057c2bf1cf0.pdf', papername='Large-Scale Semantic Co-Labeling of Image Sets', uncertainty=0.0, venue='WACV 2014', notes='')
msrc21_pp.measure(None, 83.0, 'Harmony Potentials', url='http://link.springer.com/article/10.1007%2Fs11263-011-0449-8', papername='Harmony Potentials - Fusing Local and Global Scale for Semantic Image Segmentation', uncertainty=0.0, venue='IJCV 2012', notes='per-class % / per-pixel %')
msrc21_pp.measure(None, 86.0, 'Describing the Scene as a Whole: Joint Object Detection, Scene Classification and Semantic Segmentation', url='http://ttic.uchicago.edu/~rurtasun/publications/yao_et_al_cvpr12.pdf', papername='Describing the Scene as a Whole: Joint Object Detection, Scene Classification and Semantic Segmentation', uncertainty=0.0, venue='CVPR 2012', notes='')
msrc21_pp.measure(None, 85.0, 'MPP', url='http://mediatum.ub.tum.de/doc/1175516/1175516.pdf', papername='Morphological Proximity Priors: Spatial Relationships for Semantic Segmentation', uncertainty=0.0, venue='TUM-I1222 2013', notes='')
msrc21_pp.measure(None, 86.0, 'FC CRF', url='http://graphics.stanford.edu/projects/densecrf/densecrf.pdf', papername='Efficient Inference in Fully Connected CRFs with Gaussian Edge Potentials', uncertainty=0.0, venue='NIPS 2011', notes='Strong unary used provides 76.6% / 84.0%')
msrc21_pp.measure(None, 87.0, 'HCRF+CO', url='http://research.microsoft.com/en-us/um/people/pkohli/papers/lrkt_eccv2010.pdf', papername='Graph Cut based Inference with Co-occurrence Statistics', uncertainty=0.0, venue='ECCV 2010', notes='')
msrc21_pp.measure(None, 85.0, 'Are Spatial and Global Constraints Really Necessary for Segmentation?', url='http://infoscience.epfl.ch/record/169178/files/lucchi_ICCV11.pdf', papername='Are Spatial and Global Constraints Really Necessary for Segmentation?', uncertainty=0.0, venue='ICCV 2011', notes='Several variants are examined, no single method attains the overall best results, i.e. both best per-class and per-pixel averages simultaneously. Indicated result corresponds to the method that we best on the average (per-class + per-pixel / 2). Experiment data available.')
msrc21_pp.measure(None, 82.0, 'Kernelized SSVM/CRF', url='https://infoscience.epfl.ch/record/180188/files/top.pdf', papername='Structured Image Segmentation using Kernelized Features', uncertainty=0.0, venue='ECCV 2012', notes='70 % / 73 % when using only local features (not considering global features)')
msrc21_pp.measure(None, 79.0, 'PatchMatchGraph', url='http://users.cecs.anu.edu.au/~sgould/papers/eccv12-patchGraph.pdf', papername='PatchMatchGraph: Building a Graph of Dense Patch Correspondences for Label Transfer', uncertainty=0.0, venue='ECCV 2012', notes='8% / 63.3% raw PatchMatchGraph accuracy, 72.8% / 79.0% when using Boosted CRF. Code available.')
msrc21_pp.measure(None, 78.0, 'Auto-Context', url='http://pages.ucsd.edu/~ztu/publication/pami_autocontext.pdf', papername='Auto-Context and Its Application to High-Level Vision Tasks and 3D Brain Image Segmentation', uncertainty=0.0, venue='PAMI 2010', notes='')
msrc21_pp.measure(None, 72.0, 'STF', url='http://mi.eng.cam.ac.uk/~cipolla/publications/inproceedings/2008-CVPR-semantic-texton-forests.pdf', papername='Semantic Texton Forests for Image Categorization and Segmentation', uncertainty=0.0, venue='CVPR 2008', notes='')
msrc21_pp.measure(None, 72.0, 'TextonBoost', url='http://research.microsoft.com/pubs/117885/ijcv07a.pdf', papername='TextonBoost for Image Understanding', uncertainty=0.0, venue='IJCV 2009', notes='?? / 69.6 % (per-class / per-pixel) the unaries alone (no CRF on top)')
# Handling 'Pascal VOC 2011 comp3' detection_datasets_results.html#50617363616c20564f43203230313120636f6d7033
# Skipping 40.6 mAP Fisher and VLAD with FLAIR CVPR 2014
# Handling 'Leeds Sport Poses' pose_estimation_datasets_results.html#4c656564732053706f727420506f736573
#69.2 %                  Strong Appearance and Expressive Spatial Models for Human Pose Estimation  ICCV 2013 Starting model reaches 58.1 %, improved local appearances reaches 66.9 %, and 69.2% when using the full model.
#64.3 %                                    Appearance sharing for collective human pose estimation  ACCV 2012 
#63.3 %                                                   Poselet conditioned pictorial structures  CVPR 2013 
#60.8 %                                Articulated pose estimation with flexible mixtures-of-parts  CVPR 2011 
# 55.6%           Pictorial structures revisited: People detection and articulated pose estimation  CVPR 2009 
# Handling 'Pascal VOC 2007 comp3' detection_datasets_results.html#50617363616c20564f43203230303720636f6d7033
# Skipping 22.7 mAP Ensemble of Exemplar-SVMs for Object Detection and Beyond ICCV 2011
# Skipping 27.4 mAP Measuring the objectness of image windows PAMI 2012
# Skipping 28.7 mAP Automatic discovery of meaningful object parts with latent CRFs CVPR 2010
# Skipping 29.0 mAP Object Detection with Discriminatively Trained Part Based Models PAMI 2010
# Skipping 29.6 mAP Latent Hierarchical Structural Learning for Object Detection CVPR 2010
# Skipping 32.4 mAP Deformable Part Models with Individual Part Scaling BMVC 2013
# Skipping 34.3 mAP Histograms of Sparse Codes for Object Detection CVPR 2013
# Skipping 34.3 mAP Boosted local structured HOG-LBP for object localization CVPR 2011
# Skipping 34.7 mAP Discriminatively Trained And-Or Tree Models for Object Detection CVPR 2013
# Skipping 34.7 mAP Incorporating Structural Alternatives and Sharing into Hierarchy for Multiclass Object Recognition and Detection CVPR 2013
# Skipping 34.8 mAP Color Attributes for Object Detection CVPR 2012
# Skipping 35.4 mAP Object Detection with Discriminatively Trained Part Based Models PAMI 2010
# Skipping 36.0 mAP Machine Learning Methods for Visual Object Detection archives-ouvertes 2011
# Skipping 38.7 mAP Detection Evolution with Multi-Order Contextual Co-occurrence CVPR 2013
# Skipping 40.5 mAP Segmentation Driven Object Detection with Fisher Vectors ICCV 2013
# Skipping 41.7 mAP Regionlets for Generic Object Detection ICCV 2013
# Skipping 43.7 mAP Beyond Bounding-Boxes: Learning Object Shape by Model-Driven Grouping ECCV 2012
# Handling 'Pascal VOC 2007 comp4' detection_datasets_results.html#50617363616c20564f43203230303720636f6d7034
# Skipping 59.2 mAP Spatial Pyramid Pooling in Deep Convolutional Networks for Visual Recognition ECCV 2014
# Skipping 58.5 mAP Rich feature hierarchies for accurate object detection and semantic segmentation CVPR 2014
# Skipping 29.0 mAP Multi-Component Models for Object Detection ECCV 2012
# Handling 'Pascal VOC 2010 comp3' detection_datasets_results.html#50617363616c20564f43203230313020636f6d7033
# Skipping 24.98 mAP Learning Collections of Part Models for Object Recognition CVPR 2013
# Skipping 29.4 mAP Discriminatively Trained And-Or Tree Models for Object Detection CVPR 2013
# Skipping 33.4 mAP Object Detection with Discriminatively Trained Part Based Models PAMI 2010
# Skipping 34.1 mAP Segmentation as selective search for object recognition ICCV 2011
# Skipping 35.1 mAP Selective Search for Object Recognition IJCV 2013
# Skipping 36.0 mAP Latent Hierarchical Structural Learning for Object  Detection CVPR 2010
# Skipping 36.8 mAP Object Detection by Context and Boosted HOG-LBP ECCV 2010
# Skipping 38.4 mAP Segmentation Driven Object Detection with Fisher Vectors ICCV 2013
# Skipping 39.7 mAP Regionlets for Generic Object Detection ICCV 2013
# Skipping 40.4 mAP Fisher and VLAD with FLAIR CVPR 2014
# Handling 'Pascal VOC 2010 comp4' detection_datasets_results.html#50617363616c20564f43203230313020636f6d7034
# Skipping 53.7 mAP Rich feature hierarchies for accurate object detection and semantic segmentation CVPR 2014
# Skipping 40.4 mAP Bottom-up Segmentation for Top-down Detection CVPR 2013
# Skipping 33.1 mAP Multi-Component Models for Object Detection ECCV 2012


