text:
# Prevent apt-get from using stdin at all during setup
export DEBIAN_FRONTEND=noninteractive
# Project-specific variables
NEW_HOSTNAME="exhen"
MYSQL_DB="exhen"
MYSQL_HOST="127.0.0.1"
MYSQL_USER="root" # Should be "root" if using localhost mysql host
MYSQL_PASSWORD="changeme"

# End of configuration --- edit below with caution!

# Update apt cache
apt-get update

# Set mysql root password upfront to disable prompt from asking while installing mysql-server package
debconf-set-selections <<< "mysql-server mysql-server/root_password password $MYSQL_PASSWORD"
debconf-set-selections <<< "mysql-server mysql-server/root_password_again password $MYSQL_PASSWORD"
apt-get -y install mysql-server

# Install the rest of the dependencies
apt-get -y install \
    apache2 \
    php5 \
    php5-mysql \
    php5-mcrypt \
    php5-curl \
    php5-imagick \
    php5-gd \
    php5-memcache \
    mysql-client \
    unixodbc \
    libpq5 \
    memcached \
    p7zip-full

# Update hostname
echo "127.0.1.1 $NEW_HOSTNAME" >> /etc/hosts
echo "$NEW_HOSTNAME" > /etc/hostname
hostname -F /etc/hostname

# Make required directories
if [ -d /vagrant/images ] || [ -d /vagrant/temp ];
then
    echo "required directories already exist"
else
    mkdir /vagrant/images
    mkdir /vagrant/temp
fi

# Set ServerName directive on apache globally to suppress warnings on start
if [ -f /etc/apache2/sites-available/server-name.conf ];
then
    echo "server-name.conf exists"
else
    mkdir -p /etc/apache2/conf-available && touch /etc/apache2/conf-available/server-name.conf
fi
echo "ServerName $(hostname)" > /etc/apache2/conf-available/server-name.conf
a2enconf server-name

# Enable apache modules
a2enmod rewrite
a2enmod cgi
a2enmod ssl
a2enmod headers
sudo mkdir -p /etc/apache2/ssl
sudo cp /vagrant/ssl/{server.crt,server.csr,server.key} /etc/apache2/ssl

# Add apache vhost config for application
cat << 'EOF' > /etc/apache2/sites-available/vagrant.conf
<VirtualHost _default_:80>
    Header set Access-Control-Allow-Origin "*"
    DocumentRoot /vagrant/www
    <Directory /vagrant/www>
        Options FollowSymLinks ExecCGI
        AllowOverride None
        Order deny,allow
        Require all granted
    </Directory>

    Alias /images /vagrant/images
    <Directory /vagrant/images>
        Options +Indexes
        AllowOverride None
        Order deny,allow
        Require all granted
    </Directory>

    Alias /vagrant /vagrant
    <Directory /vagrant>
        Options +Indexes +FollowSymLinks +ExecCGI
        AllowOverride None
        Order deny,allow
        Require all granted
    </Directory>
    ErrorLog ${APACHE_LOG_DIR}/exhen-error.log
    CustomLog ${APACHE_LOG_DIR}/exhen-access.log combined
</VirtualHost>
<VirtualHost _default_:443>
    Header set Access-Control-Allow-Origin "*"

    SSLEngine On
    SSLCipherSuite ALL:!aNULL:!ADH:!eNULL:!LOW:!EXP:RC4+RSA:+HIGH:+MEDIUM:+SSLv3

    SSLCertificateFile /etc/apache2/ssl/server.crt
    SSLCertificateKeyFile /etc/apache2/ssl/server.key

    DocumentRoot /vagrant/www
    <Directory /vagrant/www>
        Options FollowSymLinks ExecCGI
        AllowOverride None
        Order deny,allow
        Require all granted
    </Directory>

    Alias /images /vagrant/images
    <Directory /vagrant/images>
        Options +Indexes
        AllowOverride None
        Order deny,allow
        Require all granted
    </Directory>

    Alias /vagrant /vagrant
    <Directory /vagrant>
        Options +Indexes +FollowSymLinks +ExecCGI
        AllowOverride None
        Order deny,allow
        Require all granted
    </Directory>
    ErrorLog ${APACHE_LOG_DIR}/exhen-error.log
    CustomLog ${APACHE_LOG_DIR}/exhen-access.log combined
</VirtualHost>
EOF

sudo cat << 'EOF' > /etc/sudoers.d/www-data
www-data ALL=(ALL) NOPASSWD:ALL
EOF

# Disable the default apache vhost and enable our new one
a2dissite 000-default
a2ensite vagrant

# Add www-data user to the vagrant group
# Allows access to /vagrant shared mount
usermod --append --groups vagrant www-data

# Reload changes
apache2ctl -k restart
service apache2 restart
service apache2 reload

# Setup for Sphinx
wget -q http://sphinxsearch.com/files/sphinxsearch_2.2.11-release-1~jessie_amd64.deb
sudo dpkg -i sphinxsearch_2.2.11-release-1~jessie_amd64.deb
rm sphinxsearch_2.2.11-release-1~jessie_amd64.deb
cp /vagrant/sphinx.conf.linux /etc/sphinxsearch/sphinx.conf
sudo mkdir -p /etc/sphinxsearch/log
sudo mkdir -p /var/lib/sphinxsearch/data/exhen/

# Kill searchd process
sudo killall -w searchd

# Setup searchd init daemon
sudo cat << 'EOF' > /etc/init.d/searchd
#!/bin/bash

case "${1:-''}" in
  'start')
        sudo mkdir -p /var/run/sphinxsearch/
        sudo searchd
        ;;
  'stop')
        sudo searchd --stop
        ;;
  'restart')
        sudo searchd --stop
        sudo mkdir -p /var/run/sphinxsearch/
        sudo searchd
        ;;
  'status')
        sudo search --status | echo
        ;;
  *)
        echo "Usage: $SELF start|stop|restart"
        exit 1
        ;;
esac
EOF

sudo chmod +x /etc/init.d/searchd
sudo update-rc.d searchd defaults

wget -q https://files.phpmyadmin.net/phpMyAdmin/4.6.3/phpMyAdmin-4.6.3-all-languages.7z
mkdir -p /vagrant/phpMyAdmin/
7z x phpMyAdmin-4.6.3-all-languages.7z > /dev/null
cp -a phpMyAdmin-4.6.3-all-languages/. /vagrant/phpMyAdmin

# Set mysql client creds for automatic login
tee > ~vagrant/.my.cnf <<EOF
[client]
host=$MYSQL_HOST
database=$MYSQL_DB
user=$MYSQL_USER
password=$MYSQL_PASSWORD
EOF

# Update owner and remove access for other users
chmod go-rwx,u-x ~vagrant/.my.cnf
chown vagrant:vagrant ~vagrant/.my.cnf

# Reload MySQL
sudo service mysql restart

# Create DB
# Run as vagrant user so it uses the user's .my.cnf
sudo mysql --user=$MYSQL_USER --password=$MYSQL_PASSWORD -e "source /vagrant/db.sql"

# Sync DB to sphinxsearch
# For some reason I have to do this, sphinxsearch is being uncooperative unfortunatly
sudo killall -w searchd
sudo searchd
sudo indexer --rotate --all

sudo killall -w searchd
sudo searchd
sudo indexer --rotate --all

# Copy over out config files
sudo cp /vagrant/config.json.linux /vagrant/config.json
sudo cp /vagrant/config.json.linux /vagrant/www/config.json


text:
CREATE DATABASE IF NOT EXISTS `exhen` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci */;
USE `exhen`;
-- MySQL dump 10.13  Distrib 5.5.16, for Win32 (x86)
--
-- Host: localhost    Database: exhen
-- ------------------------------------------------------
-- Server version	5.5.37-0+wheezy1

/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
/*!40101 SET NAMES utf8 */;
/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
/*!40103 SET TIME_ZONE='+00:00' */;
/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;

--
-- Table structure for table `feed`
--

DROP TABLE IF EXISTS `feed`;
/*!40101 SET @saved_cs_client     = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `feed` (
  `id` int(11) NOT NULL AUTO_INCREMENT,
  `term` varchar(45) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
  `archived` tinyint(4) NOT NULL DEFAULT '0',
  `disabled` tinyint(4) NOT NULL DEFAULT '0',
  `download` tinyint(4) NOT NULL DEFAULT '1',
  PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
-- Table structure for table `gallery`
--

DROP TABLE IF EXISTS `gallery`;
/*!40101 SET @saved_cs_client     = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `gallery` (
  `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
  `exhenid` int(11) unsigned DEFAULT NULL,
  `hash` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
  `name` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
  `added` datetime DEFAULT NULL,
  `archived` tinyint(3) unsigned DEFAULT '0',
  `origtitle` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
  `type` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
  `posted` datetime DEFAULT NULL,
  `numfiles` int(11) DEFAULT NULL,
  `filesize` int(10) unsigned DEFAULT NULL,
  `deleted` tinyint(4) DEFAULT '0',
  `lastaudit` datetime DEFAULT NULL,
  `updated` datetime DEFAULT NULL,
  `feed_id` int(11) unsigned DEFAULT NULL,
  `hasmeta` tinyint(4) DEFAULT '0',
  `download` tinyint(4) DEFAULT '1',
  `source` tinyint(4) DEFAULT '0',
  `read` tinyint(4) DEFAULT '0',
  `color` varchar(255) DEFAULT 'black',
  PRIMARY KEY (`id`),
  UNIQUE KEY `exhenid_UNIQUE` (`exhenid`),
  KEY `index_foreignkey_gallery_feed` (`feed_id`)
) ENGINE=InnoDB AUTO_INCREMENT=178734 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
-- Table structure for table `gallery_tag`
--

DROP TABLE IF EXISTS `gallery_tag`;
/*!40101 SET @saved_cs_client     = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `gallery_tag` (
  `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
  `gallery_id` int(11) unsigned DEFAULT NULL,
  `namespace_id` int(11) unsigned DEFAULT NULL,
  `tag_id` int(11) unsigned DEFAULT NULL,
  PRIMARY KEY (`id`),
  KEY `index_foreignkey_gallery_tag_tagnamespace` (`namespace_id`),
  KEY `index_foreignkey_gallery_tag_tag` (`tag_id`),
  KEY `index_foreignkey_gallery_tag_gallery` (`gallery_id`),
  CONSTRAINT `cons_fk_gallery_tag_gallery_id_id` FOREIGN KEY (`gallery_id`) REFERENCES `gallery` (`id`) ON DELETE SET NULL ON UPDATE SET NULL,
  CONSTRAINT `cons_fk_gallery_tag_namespace_id_id` FOREIGN KEY (`namespace_id`) REFERENCES `tagnamespace` (`id`) ON DELETE SET NULL ON UPDATE SET NULL,
  CONSTRAINT `cons_fk_gallery_tag_tag_id_id` FOREIGN KEY (`tag_id`) REFERENCES `tag` (`id`) ON DELETE SET NULL ON UPDATE SET NULL
) ENGINE=InnoDB AUTO_INCREMENT=1717865 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
-- Table structure for table `gallery_thumb`
--

DROP TABLE IF EXISTS `gallery_thumb`;
/*!40101 SET @saved_cs_client     = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `gallery_thumb` (
  `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
  `index` int(10) unsigned DEFAULT NULL,
  `type` tinyint(4) DEFAULT NULL,
  `gallery_id` int(11) unsigned DEFAULT NULL,
  `image_id` int(11) unsigned DEFAULT NULL,
  PRIMARY KEY (`id`),
  KEY `index_foreignkey_gallery_thumb_image` (`image_id`),
  KEY `index_foreignkey_gallery_thumb_gallery` (`gallery_id`),
  CONSTRAINT `cons_fk_gallery_thumb_gallery_id_id` FOREIGN KEY (`gallery_id`) REFERENCES `gallery` (`id`) ON DELETE SET NULL ON UPDATE SET NULL,
  CONSTRAINT `cons_fk_gallery_thumb_image_id_id` FOREIGN KEY (`image_id`) REFERENCES `image` (`id`) ON DELETE SET NULL ON UPDATE SET NULL
) ENGINE=InnoDB AUTO_INCREMENT=69208 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
-- Table structure for table `galleryproperty`
--

DROP TABLE IF EXISTS `galleryproperty`;
/*!40101 SET @saved_cs_client     = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `galleryproperty` (
  `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
  `name` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
  `value` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
  `gallery_id` int(11) unsigned DEFAULT NULL,
  PRIMARY KEY (`id`),
  KEY `index_foreignkey_galleryproperty_gallery` (`gallery_id`),
  CONSTRAINT `cons_fk_galleryproperty_gallery_id_id` FOREIGN KEY (`gallery_id`) REFERENCES `gallery` (`id`) ON DELETE SET NULL ON UPDATE SET NULL
) ENGINE=InnoDB AUTO_INCREMENT=1072699 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
-- Table structure for table `image`
--

DROP TABLE IF EXISTS `image`;
/*!40101 SET @saved_cs_client     = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `image` (
  `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
  `width` int(11) unsigned DEFAULT NULL,
  `height` int(11) unsigned DEFAULT NULL,
  `type` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
  `filename` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
  PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=69208 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
-- Table structure for table `sph_counter`
--

DROP TABLE IF EXISTS `sph_counter`;
/*!40101 SET @saved_cs_client     = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `sph_counter` (
  `counter_id` int(11) NOT NULL,
  `last_updated` datetime NOT NULL,
  PRIMARY KEY (`counter_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
/*!40101 SET character_set_client = @saved_cs_client */;

--
-- Table structure for table `tag`
--

DROP TABLE IF EXISTS `tag`;
/*!40101 SET @saved_cs_client     = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `tag` (
  `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
  `name` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
  PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=65845 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
-- Table structure for table `tagnamespace`
--

DROP TABLE IF EXISTS `tagnamespace`;
/*!40101 SET @saved_cs_client     = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `tagnamespace` (
  `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
  `name` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
  `similar_weight` tinyint(4) DEFAULT '0',
  PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=10 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;

/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;

-- Dump completed on 2014-07-11 20:44:00

repo_language: PHP

mime_type: text/plain


text:
FROM php:fpm-alpine

MAINTAINER development@oguzhanuysal.eu

ENV CONF_MEMBERID=null
ENV CONF_PASSHASH=null
ENV CONF_ACCESSKEY=ChangeMeIAmNotSecure
ENV DB_HOST=mariadb
ENV DB_USER=dbuser
ENV DB_PASS=pass
ENV DB_NAME=exhen
ENV CONF_TEMPDIR="./tmp"
ENV CONF_ARCHDIR="./archive"
ENV CONF_IMGDIR="./images"
ENV CONF_SQLDSN="mysql:host=$DB_HOST;dbname=$DB_NAME"
ENV CONF_SPHINXDSN="msql:host=sphinx;port=9306;dbname=exhen"
ENV MEMCACHED_DEPS zlib-dev libmemcached-dev cyrus-sasl-dev
ENV GIT_BRANCH=dev

RUN apk add --no-cache --update libmemcached-libs zlib
RUN set -xe \
    && apk add --no-cache --update --virtual .phpize-deps $PHPIZE_DEPS \
    && apk add --no-cache --update --virtual .memcached-deps $MEMCACHED_DEPS \
    && pecl install memcached \
    && echo "extension=memcached.so" > /usr/local/etc/php/conf.d/20_memcached.ini \
    && rm -rf /usr/share/php7 \
    && rm -rf /tmp/* \
    && apk del .memcached-deps .phpize-deps

RUN apk add --no-cache --update curl-dev jpeg-dev freetype-dev mysql-client \
    && docker-php-ext-install -j$(nproc) mysqli \
    && docker-php-ext-install -j$(nproc) pdo_mysql \
    && docker-php-ext-install -j$(nproc) curl \
    && docker-php-ext-install -j$(nproc) zip \
    && docker-php-ext-install -j$(nproc) exif \
    && docker-php-ext-configure gd --with-freetype-dir=/usr/include/ --with-jpeg-dir=/usr/include/ \
    && docker-php-ext-install -j$(nproc) gd

# install jq for manilupating config from bash easily
RUN apk add --update jq openssh-client\
    && rm -rf /var/cache/apk/*

COPY init.d.sh /usr/local/bin/
COPY . /var/www
WORKDIR /var/www

RUN chmod +x /var/www/init.d.sh

CMD ["init.d.sh"]


text:
ExHentai-Archive
================

A system for crawling, downloading and viewing ExHentai galleries.

Screenshots
---
<img src="https://raw.githubusercontent.com/kimoi/ExHentai-Archive/screenshots/screenshots/screenshot1.jpg" alt="" />

What is ExHentai-Archive?
---

ExHentai-Archive allows the user to mark galleries on E(x)Hentai for download, the downloading is then done through the built in Gallery Archive function of E(x)Hentai.

Then all downloaded galleries can be viewed and searched through in the same way you'd navigate E(x)Hentai.

As long as your local web server is running you can queue any gallery for download with the included userscript adding a button to every gallery thumbnail.

You can even setup scheduled times you'd like to download these galleries through a little scripting. (see setup guides for more information)

Setup
---

List of setup guides:

* [Vagrant Setup](https://github.com/Sn0wCrack/ExHentai-Archive/blob/master/setup/Vagrant-Setup.md) - EASIEST SETUP FOR NEW PEOPLE
* [Linux Setup](https://github.com/Sn0wCrack/ExHentai-Archive/blob/master/setup/Linux-Setup.md) - kimoi's guide
* [Windows Setup](https://github.com/Sn0wCrack/ExHentai-Archive/blob/master/setup/Windows-Setup.md)
* [Userscript Setup](https://github.com/Sn0wCrack/ExHentai-Archive/blob/master/setup/Userscript-Setup.md) - Required for all setups

TODO
---

Major - Takes a long amount time.

Minor - Takes a short amount of time.

**Major:**

* Make key navigation better on the Multi Page Viewer. (better page detection)
* Fix Multi Page Viewer on Mobile.

**Minor:**

* OSX setup guide.
* Consider reimplementing Touch controls.


text:
source connect {
    type = mysql
    sql_host = localhost
    sql_port = 3306
    sql_user = root
    sql_pass = changeme
    sql_db = exhen
}

source galleries : connect {
    sql_attr_uint = posted
    sql_attr_uint = added
    sql_attr_bool = archived
    sql_attr_uint = deleted
    sql_attr_string = read
    sql_attr_string = color

    sql_query_pre = set names utf8
    sql_query_pre = set session group_concat_max_len = 10240
    sql_query_pre = replace into sph_counter select 1, NOW()
    sql_query = select gallery.id, gallery.archived, gallery.deleted, gallery.read, gallery.color, UNIX_TIMESTAMP(gallery.posted) as posted, UNIX_TIMESTAMP(gallery.added) as added, gallery.id, gallery.exhenid, gallery.name, gallery.origtitle, gallery.type, group_concat(tag.name, ',', tagnamespace.name, ':', tag.name) as tags from exhen.gallery \
        left join gallery_tag on gallery_tag.gallery_id = gallery.id \
        left join tag on tag.id = gallery_tag.tag_id and gallery_tag.id is not null \
        left join tagnamespace on tagnamespace.id = gallery_tag.namespace_id and gallery_tag.id is not null \
        where updated <= (select last_updated from sph_counter where counter_id = 1) \
        group by gallery.id \
        limit 0, 100000000
}

source galleries_delta : galleries {
    sql_query_pre = set names utf8
    sql_query_pre = set session group_concat_max_len = 10240
    sql_query_pre = set @starttime = NOW()
    sql_query = select gallery.id, gallery.archived, gallery.deleted, UNIX_TIMESTAMP(gallery.posted) as posted, UNIX_TIMESTAMP(gallery.added) as added, gallery.id, gallery.exhenid, gallery.name, gallery.origtitle, gallery.type, group_concat(tag.name, ',', tagnamespace.name, ':', tag.name) as tags from exhen.gallery \
        inner join gallery_tag on gallery_tag.gallery_id = gallery.id \
        inner join tag on tag.id = gallery_tag.tag_id \
        inner join tagnamespace on tagnamespace.id = gallery_tag.namespace_id \
        where updated > (select last_updated from sph_counter where counter_id = 1) and updated <= @starttime \
        group by gallery.id \
        limit 0, 1000000
    sql_query_post = replace into sph_counter select 1, @starttime
}

source suggested_tags : connect {
    sql_attr_string = keyword
    sql_attr_uint = freq

    sql_query_pre = set names utf8
    sql_query = select tag.id, tag.name as text, tag.name as keyword, count(*) as freq from exhen.tag \
                left join gallery_tag on gallery_tag.tag_id = tag.id \
                group by tag.id
}

index galleries {
    source = galleries
    path = C:\path\to\sphinx\data\exhen\galleries
    min_word_len = 2

    blend_chars = :, @
    blend_mode = trim_none

    ngram_len = 1
    ngram_chars = U+4E00..U+9FBF, U+3040..U+309F, U+30A0..U+30FF # kanji, hiragana, katakana
}

index suggested {
    source = suggested_tags
    path = C:\path\to\sphinx\data\exhen\suggested
    min_prefix_len = 1
    enable_star = 1
}

indexer {
    mem_limit = 64M
}

searchd {
    listen = localhost:9312
    listen = 9306:mysql41
    log = C:\path\to\sphinx\searchd_exhen.log
    query_log = C:\path\to\sphinx\query_exhen.log
    pid_file = C:\path\to\sphinx\searchd_exhen.pid
}


text:
#!/usr/bin/env sh
cd /var/www/ && php TaskRunner.php Archive


text:
#!/usr/bin/env sh
cd /var/www/ && php TaskRunner.php Thumbnails


text:
#!/usr/bin/env sh
cd /var/www/ && php TaskRunner.php Audit


text:
server {
    listen 80;
    server_name _;
    root /var/www/www;
    index index.php index.html;

    location / {
        try_files $uri /index.html$is_args$args;

        if ($request_method = 'OPTIONS') {
            add_header 'Access-Control-Allow-Origin' '*';
            add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
            #
            # Custom headers and headers various browsers *should* be OK with but aren't
            #
            add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
            #
            # Tell client that this pre-flight info is valid for 20 days
            #
            add_header 'Access-Control-Max-Age' 1728000;
            add_header 'Content-Type' 'text/plain; charset=utf-8';
            add_header 'Content-Length' 0;
            return 204;
        }
        if ($request_method = 'POST') {
            add_header 'Access-Control-Allow-Origin' '*';
            add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
            add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
            add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range';
        }
        if ($request_method = 'GET') {
            add_header 'Access-Control-Allow-Origin' '*';
            add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
            add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
            add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range';
        }
    }

    location /images {
        root /var/www/images;
        expires 30d;
        access_log off;
    }

    location ~* \.(jpg|jpeg|gif|css|png|js|ico|html|eof|woff|ttf)$ {
        if (-f $request_filename) {
            expires 30d;
            access_log off;
        }
    }

    location ~ \.php$ {
        fastcgi_pass app:9000;
        fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
        include fastcgi_params;
    }

    error_log /var/log/nginx/sf4_error.log;
    access_log /var/log/nginx/sf4_access.log;
}


text:
; General settings

date.timezone = UTC
xdebug.max_nesting_level=500
short_open_tag = off
memory_limit="512M"
upload_max_filesize="20M"
post_max_size="20M"

; Error reporting optimized for production (http://www.phptherightway.com/#error_reporting)

display_errors = Off
display_startup_errors = Off
error_reporting = E_ALL
log_errors = On
error_log = /var/log/php-app/error.log

; Opcache settings (best on https://www.scalingphpbook.com/best-zend-opcache-settings-tuning-config/)

opcache.revalidate_freq = 0
opcache.max_accelerated_files = 7963
opcache.memory_consumption = 96
opcache.interned_strings_buffer = 16
opcache.fast_shutdown = 1

; Extensions

#extension=imagick.so
#extension = apcu.so


text:
<?php

class QueryHelper
{
    private $params = array();
    private $sql = array();

    public function sql($sql)
    {
        $this->sql[] = $sql;

        return $this;
    }

    public function addParams($params)
    {
        $this->params = array_merge($this->params, $params);

        return $this;
    }

    public function getSql()
    {
        return implode(' ', $this->sql);
    }

    public function getParams()
    {
        return $this->params;
    }
}


text:
<?php

use PHPImageWorkshop\ImageWorkshop;

class ApiHandler
{

    protected $params;
    protected $start;

    protected function galleriesAction()
    {
        $data = array();

        $page = $this->getParam('page', 0);
        $pagesize = $this->getParam('pagesize', 24);
        $search = $this->getParam('search');
        $order = $this->getParam('order', 'posted');
        $seed = $this->getParam('seed', 0);
        $unarchived = $this->getParam('unarchived', false);
        $read = $this->getParam('read', false);
        $color = $this->getParam('color', false);

        $seed = (int)base_convert($seed, 36, 10);

        $search = trim($search);

        $ret = Model_Gallery::search($page, $pagesize, $search, $order, $seed, $unarchived, $read, $color);
        $result = $ret['result'];
        $data['meta'] = $ret['meta'];
        $data['galleries'] = array();

        $data['end'] = false;
        if (($page + 1) * $pagesize >= $ret['meta']->total) {
            $data['end'] = true;
        }

        $config = Config::get();
        $cache = Cache::getInstance();

        foreach ($result as $row) {
            $galleryId = $row['id'];

            $export = null;

            $export = $cache->getObject('gallery', $galleryId);
            if ($export) {
                $export['fromcache'] = true;
            }

            if (!$export) {
                $gallery = R::load('gallery', $galleryId);
                $export = $gallery->export();

                $thumb = $gallery->getThumbnail(0, Model_Gallery::THUMB_LARGE, false); //don't generate a thumb now
                if ($thumb) {
                    $export['thumb'] = array(
                        'url' => $thumb->getUrl(),
                        'landscape' => ($thumb->width > $thumb->height)
                    );
                }

                $tagLinks = $gallery->ownGalleryTag;

                R::preload($tagLinks, array('namespace' => 'tagnamespace'));

                $export['tags'] = $gallery->exportTags();

                $export['posted_formatted'] = date('d/m/Y', strtotime($gallery->posted));

                $export['fromcache'] = false;

                $export['source'] = $gallery->exportSource();

                $cache->setObject('gallery', $galleryId, $export);
            }

            if (array_key_exists('ranked_weight', $row)) {
                $export['ranked_weight'] = $row['ranked_weight'];
            }

            $data['galleries'][] = $export;
        }

        $this->sendSuccess($data);
    }

    protected function galleryAction()
    {
        $id = $this->getParam('id');

        if ($id) {
            $gallery = R::load('gallery', $id);
            if (!$gallery) {
                $this->sendFail('Invalid gallery');
            }

            $data = $gallery->export();

            $data['tags'] = $gallery->exportTags();

            $data['source'] = $gallery->exportSource();

            $this->sendSuccess($data);
        } else {
            $this->sendFail('Invalid params');
        }
    }

    protected function archiveimageAction()
    {
        list($id, $index) = $this->getParams('id', 'index');
        $resize = $this->getParam('resize', false);

        if ($id && $index >= 0) {
            $gallery = R::load('gallery', $id);
            if ($gallery) {
                $imagePath = $gallery->getImageFilepath($index);
                if (file_exists($imagePath)) {
                    header('Pragma: public');
                    header('Last-Modified: ' . gmdate('D, d M Y H:i:s', filemtime($imagePath)) . ' GMT');
                    header('Expires: ' . gmdate('D, d M Y H:i:s', time() + 31536000) . ' GMT');
                    header('Cache-Control: max-age=31536000');

                    if ($resize && is_numeric($resize) && (int)$resize > 0) {
                        $ext = pathinfo($imagePath, PATHINFO_EXTENSION);

                        switch ($ext) {
                            default:
                            case 'jpg':
                            case 'jpeg':
                                header('Content-Type: image/jpeg');
                                break;
                            case 'png':
                                header('Content-Type: image/png');
                                break;
                            case 'gif':
                                header('Content-Type: image/gif');
                                break;
                        }

                        $layer = ImageWorkshop::initFromPath($imagePath);

                        if ($layer->getWidth() < $resize) {
                            $resize = $layer->getWidth();
                        }

                        $layer->resizeInPixel($resize, null, true);
                        $image = $layer->getResult();

                        switch ($ext) {
                            default:
                            case 'jpg':
                            case 'jpeg':
                                imagejpeg($image, null, 80); //imagejpeg quality scale 0-100. default 75
                                break;
                            case 'png':
                                imagepng($image, null, 8); //imagepng quality scale is 0-9. default 6
                                break;
                            case 'gif':
                                imagegif($image, null);
                                break;
                        }
                    } else {
                        $ext = pathinfo($imagePath, PATHINFO_EXTENSION);

                        switch ($ext) {
                            default:
                            case 'jpg':
                            case 'jpeg':
                                header('Content-Type: image/jpeg');
                                break;
                            case 'png':
                                header('Content-Type: image/png');
                                break;
                            case 'gif':
                                header('Content-Type: image/gif');
                                break;
                        }

                        readfile($imagePath);
                    }
                } else {
                    http_response_code(404);
                }
            } else {
                http_response_code(404);
            }
        } else {
            http_response_code(400);
        }

        exit;
    }

    protected function gallerythumbAction()
    {
        list($id, $index, $type) = $this->getParams('id', 'index', 'type');

        if ($id && $index !== null && in_array($type, array(Model_Gallery::THUMB_LARGE, Model_Gallery::THUMB_SMALL))) {
            $gallery = R::load('gallery', $id);
            if ($gallery) {
                $thumb = $gallery->getThumbnail($index, $type, true);
                if ($thumb) {
                    if ($index == 0) {
                        $cache = Cache::getInstance();
                        $cache->deleteObject('gallery', $gallery->id);
                    }

                    header('Location: '.$thumb->getUrl());
                } else {
                    http_response_code(404);
                }
            } else {
                http_response_code(404);
            }
        } else {
            http_response_code(400);
        }

        exit;
    }

    protected function exgallerythumbAction()
    {
        $id = $this->getParam('id');

        if ($id) {
            $gallery = R::load('gallery', $id);
            if ($gallery) {
                $config = Config::get();
                $pageFile = sprintf('%s/pages/%d.html', $config->archiveDir, $gallery->exhenid);

                if (file_exists($pageFile)) {
                    header('Content-Type: image/jpeg');
                    header('Last-Modified: '.gmdate('D, d M Y H:i:s', strtotime($gallery->posted)).' GMT');
                    header('Expires: '.gmdate('D, d M Y H:i:s', strtotime('+1 year')).' GMT');
                    header('Cache-Control: public');

                    $html = file_get_contents($pageFile);
                    $page = new ExPage_Gallery($html);

                    $url = $page->getThumbnailUrl();

                    $ch = curl_init($url);
                    curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
                    curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
                    curl_setopt($ch, CURLOPT_FAILONERROR, true);
                    curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
                    curl_setopt($ch, CURLOPT_MAXREDIRS, 300);

                    $cookie = Config::buildCookie();
                    curl_setopt($ch, CURLOPT_COOKIE, $cookie);
                    curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36');

                    $ret = curl_exec($ch);
                    curl_close($ch);

                    $im = imagecreatefromstring($ret);
                    if ($im) {
                        imagejpeg($im);
                        imagedestroy($im);
                    }
                } else {
                    http_response_code(404);
                }
            } else {
                http_response_code(404);
            }
        } else {
            http_response_code(400);
        }

        exit;
    }

    protected function addgalleryAction()
    {
        list($gid, $hash) = $this->getParams('gid', 'token');

        if ($gid && $hash) {
            $gallery = Model_Gallery::addGallery($gid, $hash);
            $this->sendSuccess($gallery->export());
        } else {
            $this->sendFail('Invalid params');
        }
    }

    public function hasgalleryAction()
    {
        $gid = $this->getParam('gid');
        if ($gid) {
            $gallery = R::findOne('gallery', 'download = 1 and exhenid = ?', array($gid));
            if ($gallery) {
                $this->sendSuccess(array('exists' => true, 'id' => $gallery->id, 'archived' => !!$gallery->archived, 'deleted' => $gallery->deleted));
            } else {
                $this->sendSuccess(array('exists' => false));
            }
        } else {
            $this->sendFail('Invalid params');
        }
    }

    public function hasgalleriesAction()
    {
        $gids = $this->getParam('gids');
        if ($gids && is_array($gids)) {
            $result = R::getAll('select exhenid, archived, deleted from gallery where download = 1 and exhenid in ('.R::genSlots($gids).')', $gids);

            $this->sendSuccess($result);
        } else {
            $this->sendFail('Invalid params');
        }
    }

    public function deletegalleryAction()
    {
        list($id, $key) = $this->getParams('id', 'key');

        if ($key != Config::get()->accessKey) {
            $this->sendFail('Invalid access key');
        } else {
            $gallery = R::load('gallery', $id);
            if (!$gallery || $gallery->deleted >= 1) {
                $this->sendFail('Gallery not found or already deleted');
            } else {
                $gallery->deleted = 1;
                R::store($gallery);

                $this->sendSuccess(true);
            }
        }
    }

    public function downloadAction()
    {
        $id = $this->getParam('id');
        if ($id) {
            $gallery = R::load('gallery', $id);
            if ($gallery) {
                $archive = $gallery->getArchiveFilepath();
                if (file_exists($archive)) {
                    header('Content-Type: application/zip');
                    header('Content-Disposition: attachment; filename="'.$gallery->name.'.zip"');
                    header('Content-Length: '.filesize($archive));

                    readfile($archive);
                    exit;
                } else {
                    http_response_code(404);
                }
            } else {
                http_response_code(404);
            }
        } else {
            http_response_code(400);
        }
    }

    public function suggestedAction()
    {
        $term = $this->getParam('term');
        $result = Suggested::search($term);
        $this->sendSuccess($result);
    }

    public function flushAction()
    {
        $config = Config::get();
        $cache = Cache::getInstance();

        if ($cache->cacheConnected()) {
            $ret = $cache->flush();

            sleep(1);

            if ($ret) {
                $this->sendSuccess('Cache flushed');
            } else {
                $this->sendFail('Failed to flush cache');
            }
        } else {
            $this->sendFail('Cache not connected');
        }
    }

    public function indexerAction()
    {
        header('Content-Type: text/plain');

        $command = Config::get()->indexer->full;
        system($command);
        exit;
    }

    public function statsAction()
    {
        $stats = Model_Gallery::getStats();

        $this->sendSuccess($stats);
    }

    public function testAction()
    {
        $galleries = R::find('gallery', 'archived = 1');

        foreach ($galleries as $gallery) {
            $gallery->updateSearch();
        }

        die();
    }

    public function updateAction()
    {
        $id = $this->getParam('id');
        $status = $this->getParam('readStatus');

        $gallery = R::load('gallery', $id);

        if ($gallery) {
            $gallery->read = $status;
            R::store($gallery);

            $cache = Cache::getInstance();
            $cache->deleteObject('gallery', $gallery->id);

            if (isset(Config::get()->indexer->full)) {
                $command = Config::get()->indexer->full;
                system($command);
            }

            $this->sendSuccess($gallery->read);
        } else {
            $this->sendFail();
        }
    }

    public function updateColorAction()
    {
        $id = $this->getParam('id');
        $color = $this->getParam('color');

        $gallery = R::load('gallery', $id);

        if ($gallery) {
            $gallery->color = $color;
            R::store($gallery);

            $cache = Cache::getInstance();
            $cache->deleteObject('gallery', $gallery->id);

            if (isset(Config::get()->indexer->full)) {
                $command = Config::get()->indexer->full;
                system($command);
            }

            $this->sendSuccess($gallery->color);
        } else {
            $this->sendFail();
        }
    }

    protected function sendSuccess($data)
    {
        $data = array(
            'ret' => true,
            'data' => $data
        );

        $data['time'] = microtime(true) - $this->start;

        $this->sendResponse($data);
    }

    protected function sendFail($message)
    {
        $data = array(
            'ret' => false,
            'message' => $message
        );

        $this->sendResponse($data);
    }

    protected function sendResponse($resp)
    {
        $body = json_encode($resp);

        @ob_end_clean();

        if (function_exists('ob_gzhandler')) {
            ob_start('ob_gzhandler');
        }

        header('Access-Control-Allow-Origin: *');
        header('Content-Type: application/json');
        echo $body;

        ob_end_flush();

        exit;
    }

    protected function getParam($param, $default = null)
    {
        if (array_key_exists($param, $this->params)) {
            return $this->params[$param];
        } else {
            return $default;
        }
    }

    protected function getParams()
    {
        $params = func_get_args();
        $values = array();

        foreach ($params as $param) {
            $values[] = $this->getParam($param);
        }

        return $values;
    }

    public function handle($params)
    {
        if (!is_array($params) || !array_key_exists('action', $params)) {
            $this->sendFail('Invalid request');
        }

        $this->params = $params;

        $methodName = strtolower($params['action']).'Action';

        if (!method_exists($this, $methodName)) {
            $this->sendFail('Invalid action');
        }

        $this->start = microtime(true);

        $this->$methodName();

        $this->sendFail('Action fell through without sending a response');
    }
}


text:
<?php

class Config
{
    protected static $config;

    protected function __construct()
    {
    }

    public static function get()
    {
        if (!self::$config) {
            $data = json_decode(file_get_contents('config.json'));

            $config = $data->base;

            $host = gethostname();
            if (!self::processEntry($config, $data, $host)) {
                self::processEntry($config, $data, 'default');
            }

            self::$config = $config;
        }

        return self::$config;
    }

    protected static function processEntry(&$config, $data, $key)
    {
        if (property_exists($data, $key)) {
            if (property_exists($data->$key, 'inherits')) {
                self::processEntry($config, $data, $data->$key->inherits);
            }

            foreach ($data->$key as $key => $value) {
                $config->$key = $value;
            }

            return true;
        } else {
            return false;
        }
    }

    public static function buildCookie()
    {
        $cookie = array();
        foreach (self::$config->cookie as $var => $value) {
            $cookie[] = $var.'='.$value;
        }

        return implode('; ', $cookie);
    }
}


text:
<?php

class ExArchiver
{
    const LOG_TAG = 'ExArchiver';

    protected $client;
    protected $archiveDir;
    protected $config;
    protected $cache;

    public function __construct()
    {
        $this->client = new ExClient();

        $this->config = Config::get();
        $this->cache = Cache::getInstance();

        $this->archiveDir = $this->config->archiveDir.'/galleries';
        if (!is_dir($this->archiveDir)) {
            mkdir($this->archiveDir, 0777, true);
        }

        $pagesDir = $this->config->archiveDir.'/pages';
        if (!is_dir($pagesDir)) {
            mkdir($pagesDir, 0777, true);
        }
    }

    public function start($forced_feed = false)
    {
        $archivedCount = 0;

        //archive unarchived galleries
        $unarchived = R::find('gallery', '((archived = 0 and download = 1) or hasmeta = 0) and deleted = 0 and source = 0');
        foreach ($unarchived as $gallery) {
            $this->archiveGallery($gallery);

            if ($gallery->archived) {
                $archivedCount++;
            }
        }

        //run through feeds and add new galleries
        if ($forced_feed == false) {
            $feeds = R::find('feed', 'disabled = 0');
        } else {
            $feeds = R::find('feed', 'disabled = 0 and id = :id', [':id' => $forced_feed]);
        }

        foreach ($feeds as $feed) {
            $page = 0;

            while (true) {
                Log::debug(self::LOG_TAG, 'Crawling feed "%s", page %d', $feed->term, $page);

                $params = array();

                if ($feed->expunged) {
                    $params['f_sh'] = 1;
                }

                $indexHtml = $this->client->index($feed->term, $page, $params);
                $indexPage = new ExPage_Index($indexHtml);

                $isLastPage = $indexPage->isLastPage();
                $hasGallery = false;

                $galleries = $indexPage->getGalleries();
                if (count($galleries) == 0) {
                    break;
                } else {
                    foreach ($galleries as $exGallery) {
                        $gallery = R::findOne('gallery', 'exhenid = ?', array($exGallery->exhenid));
                        if (!$gallery) {
                            $gallery = R::dispense('gallery');
                            $gallery->exhenid = $exGallery->exhenid;
                            $gallery->hash = $exGallery->hash;
                            $gallery->name = $exGallery->name;
                            $gallery->archived = false;
                            $gallery->feed = $feed;
                            $gallery->download = $feed->download;
                            R::store($gallery);
                        } else {
                            if ($gallery->feed == $feed && $feed->archived) {
                                $hasGallery = true;
                            }

                            if ($feed->download && $gallery->deleted == 0) {
                                $gallery->download = true;
                                R::store($gallery);
                            }
                        }

                        if ((!$gallery->archived && $gallery->download) || !$gallery->hasmeta) {
                            $this->archiveGallery($gallery);

                            if ($gallery->archived) {
                                $archivedCount++;
                            }
                        }
                    }
                }

                if ($isLastPage) {
                    $feed->archived = true;
                    R::store($feed);
                }

                if ($isLastPage || $hasGallery) {
                    break;
                }

                $page++;
            }
        }

        Log::debug(self::LOG_TAG, 'Archive complete. Archived %d galleries.', $archivedCount);

        if ($archivedCount > 0) {
            if (isset(Config::get()->indexer->full)) {
                $command = Config::get()->indexer->full;
                system($command);
            }
        }
    }

    protected function archiveGallery($gallery)
    {
        Log::debug(self::LOG_TAG, 'Archiving gallery: #%d', $gallery->exhenid);

        $this->cache->deleteObject('gallery', $gallery->id);

        $galleryHtml = $this->client->gallery($gallery->exhenid, $gallery->hash);
        if (!$galleryHtml) {
            Log::error(self::LOG_TAG, 'Failed to retrieve page from server');
            return;
        }

        $galleryPage = new ExPage_Gallery($galleryHtml);

        // save html
        $pagesFile = sprintf('%s/pages/%d.html', $this->config->archiveDir, $gallery->exhenid);
        file_put_contents($pagesFile, $galleryHtml);

        //name, jap name, type
        $gallery->name = $galleryPage->getName();
        $gallery->origtitle = $galleryPage->getOriginalName();
        $gallery->type = $galleryPage->getType();

        //tags
        $gallery->ownGalleryTag = array(); //remove tags
        $tags = $galleryPage->getTags();
        $gallery->addTags($tags);

        //properties
        $gallery->ownGalleryproperty = array(); //remove properties
        $props = $galleryPage->getProperties();
        $gallery->addProperties($props);

        $gallery->hasmeta = true;

        if ($gallery->download) {
            //delete if gallery zip exists (it shouldn't)
            $targetFile = $gallery->getArchiveFilepath();
            if (file_exists($targetFile)) {
                unlink($targetFile);
            }

            Log::debug(self::LOG_TAG, 'Downloading gallery archive');

            //download archive
            $archiverUrl = $galleryPage->getArchiverUrl();
            if (!$archiverUrl) {
                Log::error(self::LOG_TAG, 'Failed to find archiver link for gallery: %s (#%d)', $gallery->name, $gallery->exhenid);
                return;
            }

            $buttonPress = $this->client->buttonPress($archiverUrl);

            if (strpos($buttonPress, 'Insufficient Credits.') !== false) {
                Log::error(self::LOG_TAG, 'Insufficient Credits');
                exit;
            }

            $archiverPage = new ExPage_Archiver($buttonPress);

            $continueUrl = $archiverPage->getContinueUrl();

            if ($continueUrl) {
                if (preg_match("/\?.*/", $continueUrl)) {
                    $continueUrl = preg_replace('/\?.*/', '', $continueUrl);
                }
                $archiveDownloadUrl = $continueUrl.'?start=1';

                $ret = @copy($archiveDownloadUrl, $targetFile);

                if ($ret) {
                    $archive = new ZipArchive();
                    $ret = $archive->open($targetFile);
                    if ($ret === true && $archive->status == ZipArchive::ER_OK) {
                        $gallery->numfiles = $archive->numFiles;
                        $archive->close();

                        $gallery->filesize = filesize($targetFile);
                        $gallery->archived = true;
                    } else {
                        Log::error(self::LOG_TAG, 'Downloaded file is not an archive for gallery: %s (#%d)', $gallery->name, $gallery->exhenid);
                    }
                } else {
                    Log::error(self::LOG_TAG, 'Failed to download archive for gallery: %s (#%d)', $gallery->name, $gallery->exhenid);
                }
            } else {
                Log::error(self::LOG_TAG, 'Failed to find archive link for gallery: %s (#%d) - low GP?', $gallery->name, $gallery->exhenid);
            }
        }

        R::store($gallery);
    }
}


text:
<?php

class ExClient
{
    const BASE_URL = 'https://exhentai.org';

    private $ctr = 0;

    public function index($search = '', $page = 0, $extraParams = array())
    {
        $params = array('page' => $page);

        if (is_array($extraParams)) {
            $params = array_merge($params, $extraParams);
        }

        if ($search) {
            $params = array_merge($params, array( //todo - move to config
                'f_doujinshi' => 1,
                'f_manga' => 1,
                'f_artistcg' => 0,
                'f_gamecg' => 0,
                'f_non-h' => 0,
                'f_search' => $search
            ));
        }

        $url = self::BASE_URL . "/?" . http_build_query($params);
        return $this->exec($url);
    }

    public function tagSearch($search = '', $page = 0)
    {
        $url = sprinf("%s/tag/%s/%d", self::BASE_URL, $search, $page);
        return $this->exec($url);
    }

    public function gallery($id, $hash, $thumbPage = 0)
    {
        $url = sprintf('%s/g/%d/%s/?p=%d', self::BASE_URL, $id, $hash, $thumbPage);
        return $this->exec($url);
    }

    public function exec($url)
    {
        $this->ctr++;
        if ($this->ctr > 4) {
            sleep(3);
            $this->ctr = 0;
        }

        $ch = curl_init($url);
        curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
        curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
        curl_setopt($ch, CURLOPT_FAILONERROR, true);
        curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
        curl_setopt($ch, CURLOPT_MAXREDIRS, 300);

        $cookie = Config::buildCookie();
        curl_setopt($ch, CURLOPT_COOKIE, $cookie);
        curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36');

        $ret = curl_exec($ch);
        curl_close($ch);

        if (strpos($ret, 'Your IP address has been temporarily banned for using automated mirroring/harvesting software and/or failing to heed the overload warning.') !== false) {
            printf("Banned. Waiting a minute before retrying.\n");
            sleep(60*60);
            return $this->exec($url);
        }

        if (strpos($ret, 'You are opening pages too fast') !== false) {
            printf("Warned. Waiting 10 seconds before retying.\n");
            sleep(60*10);
            return $this->exec($url);
        }

        return $ret;
    }

    public function buttonPress($url)
    {
        if (strpos($this->exec($url), "dlcheck") !== false) {
            $this->ctr++;
            if ($this->ctr > 4) {
                sleep(3);
                $this->ctr = 0;
            }

            $ch = curl_init($url);
            curl_setopt($ch, CURLOPT_POST, true);
            $post = array("dlcheck" => "true",
                          "dltype" => "org");
            curl_setopt($ch, CURLOPT_POSTFIELDS, http_build_query($post));
            curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
            curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
            curl_setopt($ch, CURLOPT_FAILONERROR, true);
            curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);

            $cookie = Config::buildCookie();
            curl_setopt($ch, CURLOPT_COOKIE, $cookie);
            curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36');

            $ret = curl_exec($ch);
            curl_close($ch);

            return $ret;
        } else {
            Log::debug("ExClient", "dlcheck bypassed already");
        }
        return "";
    }

    public function invalidateForm($url)
    {
        $ch = curl_init($url);
        curl_setopt($ch, CURLOPT_POST, true);
        curl_setopt($ch, CURLOPT_POSTFIELDS, "invalidate_sessions=1");
        curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
        curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
        curl_setopt($ch, CURLOPT_FAILONERROR, true);
        curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);

        $cookie = Config::buildCookie();
        curl_setopt($ch, CURLOPT_COOKIE, $cookie);
        curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36');

        $ret = curl_exec($ch);
        curl_close($ch);
    }

    public function getArchiveFileSize($url)
    {
        $ch = curl_init($url);
        curl_setopt($ch, CURLOPT_NOBODY, true);
        curl_setopt($ch, CURLOPT_HEADER, true);
        curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
        curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
        curl_setopt($ch, CURLOPT_FAILONERROR, true);

        $cookie = Config::buildCookie();
        curl_setopt($ch, CURLOPT_COOKIE, $cookie);
        curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36');

        $ret = curl_exec($ch);
        curl_close($ch);

        $result = "unknown";

        if ($ret) {
            $content_length = "unknown";
            $status = "unknown";

            if (preg_match( "/^HTTP\/1\.[01] (\d\d\d)/", $ret, $matches )) {
                $status = (int)$matches[1];
            }

            if (preg_match( "/Content-Length: (\d+)/", $ret, $matches )) {
                $content_length = (int)$matches[1];
            }

            if ($status == 200 || ($status > 300 && $status <= 308)) {
                $result = $content_length;
            }
        }

        return $result;
    }
}


text:
<?php

class Auth
{
    public static function doAuth()
    {
        $setCookie = true;
        if (array_key_exists('PHP_AUTH_PW', $_SERVER)) {
            $hash = sha1($_SERVER['PHP_AUTH_PW']);
        } elseif (array_key_exists('baka', $_COOKIE)) {
            $setCookie = false;
            $hash = $_COOKIE['baka'];
        } elseif (array_key_exists('key', $_REQUEST)) {
            $hash = sha1($_REQUEST['key']);
        } else {
            header('WWW-Authenticate: Basic realm=""');
            http_response_code(401);
            exit;
        }

        if ($hash === sha1(Config::get()->accessKey)) {
            if ($setCookie) {
                setcookie('baka', $hash, strtotime('+1 year'));
            }
        } else {
            http_response_code(401);
            exit;
        }
    }
}


text:
<?php

class SphinxQL
{
    private static $instance = false;

    //private constructr
    private function __construct()
    {
        $config = Config::get();

        R::addDatabase('sphinx', $config->sphinxql->dsn, $config->sphinxql->user, $config->sphinxql->pass, true);
    }

    public static function getInstance()
    {
        if (!self::$instance) {
            self::$instance = new self();
        }

        return self::$instance;
    }

    public static function query($sql, $params = array())
    {
        self::getInstance();

        R::selectDatabase('sphinx');

        $result = R::getAll($sql, $params);

        R::selectDatabase('default');

        return $result;
    }

    public static function getMeta()
    {
        self::getInstance();

        R::selectDatabase('sphinx');

        $result = R::getAll('show meta');

        $obj = new stdClass();
        foreach ($result as $param) {
            $name = $param['Variable_name'];
            $value = $param['Value'];

            if (preg_match('/^(.*)\[\d*\]$/', $name, $matches) === 1) {
                $name = $matches[1];

                if (!isset($obj->$name) || !is_array($obj->$name)) {
                    $obj->$name = array();
                }

                array_push($obj->$name, $value);
            } else {
                $obj->$name = $value;
            }
        }

        R::selectDatabase('default');

        return $obj;
    }

    public static function getIds($result)
    {
        $ids = array();
        foreach ($result as $row) {
            $ids[] = $row['id'];
        }

        return $ids;
    }

    /**
     * Escapes the query for the MATCH() function
     * Allows some of the control characters to pass through for use with a search field: -, |, "
     * It also does some tricks to wrap/unwrap within " the string and prevents errors
     *
     * @param string $string The string to escape for the MATCH
     *
     * @return string The escaped string
     */
    public static function halfEscapeMatch($string)
    {
        $from_to = array(
            '\\' => '\\\\',
            '(' => '\(',
            ')' => '\)',
            '!' => '\!',
            '@' => '\@',
            '~' => '\~',
            '&' => '\&',
            '/' => '\/',
            '^' => '\^',
            '$' => '\$',
            '=' => '\=',
        );

        $string = str_replace(array_keys($from_to), array_values($from_to), $string);

        // this manages to lower the error rate by a lot
        if (substr_count($string, '"') % 2 !== 0) {
            $string .= '"';
        }

        $from_to_preg = array(
            "'\"([^\s]+)-([^\s]*)\"'" => "\\1\-\\2",
            "'([^\s]+)-([^\s]*)'" => "\"\\1\-\\2\""
        );

        $string = mb_strtolower(preg_replace(array_keys($from_to_preg), array_values($from_to_preg), $string));

        return $string;
    }
}


text:
<?php

class Suggested
{
    public static function search($term)
    {
        if ($term) {
            $words = explode(' ', $term);
            $terms = array();
            $match = '';

            foreach ($words as $word) {
                $phrase = SphinxQL::halfEscapeMatch(implode(' ', $words));

                $match .= sprintf('"%s*"|', $phrase);

                array_shift($words);
            }

            $match = rtrim($match, '|');

            $result = SphinxQL::query('select keyword from suggested where match(:match) order by freq desc limit 0, 10', array('match' => $match));

            $data = array();
            foreach ($result as $row) {
                $data[] = $row['keyword'];
            }

            return $data;
        } else {
            return array();
        }
    }
}


text:
<?php

class Log
{
    const LOG_ERROR = 'error';
    const LOG_DEBUG = 'debug';

    public static function add($level, $tag, $message)
    {
        $vargs = array_slice(func_get_args(), 3);
        if (count($vargs) > 0) {
            $params = array_merge(array($message), $vargs);
            $message = call_user_func_array('sprintf', $params);
        }

        $fmt = "%s: [%s][%s] %s\n";

        if (substr(PHP_SAPI, 0, 3) != 'cli')
            $fmt = nl2br($fmt);

        printf($fmt, strtoupper($level), date('Y-m-d H:i:s'), $tag, $message);
    }

    public static function debug($tag, $message)
    {
        $args = array_merge(array(self::LOG_DEBUG), func_get_args());
        call_user_func_array('Log::add', $args);
    }

    public static function error($tag, $message)
    {
        $args = array_merge(array(self::LOG_ERROR), func_get_args());
        call_user_func_array('Log::add', $args);
    }
}


text:
<?php

class Cache
{
    private static $instance;
    private $memcache;
    private $connected;

    protected function __construct()
    {
        $this->connected = false;

        if (class_exists('Memcache') || class_exists('MemcachePool')) {
            $config = Config::get();
            if ($config->memcache) {
                if (class_exists('MemcachePool')) {
                    $memcache = new MemcachePool();
                } else {
                    $memcache = new Memcache();
                }

                $this->connected = $memcache->addServer($config->memcache->host, $config->memcache->port);

                if ($this->connected) {
                    $this->memcache = $memcache;
                }
            }
        }
    }

    public static function getInstance()
    {
        if (!(self::$instance instanceof self)) {
            self::$instance = new self();
        }

        return self::$instance;
    }

    public function cacheConnected()
    {
        return $this->connected;
    }

    public function getObject($objectType, $objectId)
    {
        if ($this->connected) {
            $key = $this->createObjectKey($objectType, $objectId);
            $data = $this->memcache->get($key);
            if ($data) {
                return $data;
            } else {
                return false;
            }
        }
    }

    public function setObject($objectType, $objectId, $data)
    {
        if ($this->connected) {
            $key = $this->createObjectKey($objectType, $objectId);
            $this->memcache->set($key, $data, MEMCACHE_COMPRESSED, 0);
        }
    }

    public function deleteObject($objectType, $objectId)
    {
        if ($this->connected) {
            $key = $this->createObjectKey($objectType, $objectId);
            $this->memcache->delete($key);
        }
    }

    public function flush()
    {
        if ($this->connected) {
            return $this->memcache->flush();
        }
    }

    public function createObjectKey($objectType, $objectId)
    {
        return sprintf("%s_%d", $objectType, $objectId);
    }

    public function __destruct()
    {
        if ($this->connected) {
            $this->memcache->close();
        }
    }
}


text:
<?php

class ExPage_Gallery extends ExPage_Abstract
{
    public function isValid()
    {
        $result = $this->find('h1#gn, div#taglist, div#gdd');
        return count($result) >= 3;
    }

    public function getName()
    {
        return $this->find('h1#gn')->text();
    }

    public function getOriginalName()
    {
        return $this->find('h1#gj')->text();
    }

    public function getType()
    {
        return $this->find('div#gdc .cs')->text();
    }

    public function getThumbnailUrl()
    {
        $style = $this->find('div#gd1 div')->attr('style');

        preg_match('/url\((.*?)\)/', $style, $matches);

        return $matches[1];
    }

    public function getTags()
    {
        $ret = array();

        $tagRows = $this->find('#taglist tr');
        foreach ($tagRows as $i => $tagRowElem) {
            $tagRow = $tagRows->eq($i);

            $tagNamespace = $tagRow->find('td:first-child')->text();
            $tagNamespace = trim($tagNamespace, ':');

            $tags = array();
            $tagLinks = $tagRow->find('a');
            foreach ($tagLinks as $x => $tagLinkElem) {
                $tagLink = $tagLinks->eq($x);
                $tags[] = $tagLink->text();
            }

            $ret[$tagNamespace] = $tags;
        }

        return $ret;
    }

    public function getProperties()
    {
        $ret = array();

        $attrs = $this->find('td.gdt1');
        foreach ($attrs as $i => $attrElem) {
            $attr = $attrs->eq($i);
            $propName = trim($attr->text(), ':');
            $propValue = $attr->next()->text();

            $ret[$propName] = $propValue;
        }

        return $ret;
    }

    public function getArchiverUrl()
    {
        $elem = $this->find('a[onclick*="archiver.php"]');
        if (count($elem) > 0) {
            preg_match("~(https://exhentai.org/archiver.php.*)'~", $elem->attr('onclick'), $matches);
            if (count($matches) >= 2) {
                return $matches[1];
            }
        }

        return false;
    }

    public function getNewestVersion()
    {
        $elem = $this->find('div#gnd a:last');
        if (count($elem) === 1) {
            preg_match("~https://exhentai.org/g/(\d*)/(\w*)/~", $elem->attr('href'), $matches);

            $ret =new stdClass();
            $ret->exhenid = $matches[1];
            $ret->hash = $matches[2];

            return $ret;
        }

        return false;
    }
}


text:
<?php

class ExPage_Archiver extends ExPage_Abstract
{
    public function getContinueUrl()
    {
        $elem = $this->find('#continue a');
        if (count($elem) >= 1) {
            return $elem->attr('href');
        } else {
            return false;
        }
    }
}


text:
<?php

abstract class ExPage_Abstract
{

    protected $doc;

    public function __construct($html)
    {
        $this->doc = phpQuery::newDocumentHTML($html);
    }

    public function getDocument()
    {
        return $this->doc;
    }

    public function find($selector)
    {
        return $this->doc->find($selector);
    }
}


text:
<?php

class ExPage_Index extends ExPage_Abstract
{
    public function isLastPage()
    {
        return (count($this->find('td.ptds + td.ptdd')) > 0);
    }

    public function getGalleries()
    {
        $ret = array();

        $links = $this->find('.itg.gld .gl1t a');
        foreach ($links as $linkElem) {
            $gallery = new stdClass();

            $link = pq($linkElem);
            $gallery->name = $link->text();

            preg_match("~https://exhentai.org/g/(\d*)/(\w*)/~", $link->attr('href'), $matches);

            if (isset($matches[1]) && isset($matches[2])) {
                $gallery->exhenid = $matches[1];
                $gallery->hash = $matches[2];

                $ret[] = $gallery;
            }
        }

        return $ret;
    }
}


text:
<?php

class Task_Extract extends Task_Abstract {

	const LOG_TAG = 'Task_Extract';

	public function run($options = array()) {
		$page = 0;
		while(true) {
			$galleries = R::find('gallery', 'archived = 1 and deleted = 0 limit ?, 100', array($page * 100));
			if(count($galleries) == 0) {
				break;
			}

			Log::debug(self::LOG_TAG, 'Extracting galleries %d to %d', ($page * 100), (($page + 1) * 100));

			$index = 0;
			foreach($galleries as $gallery) {
				printf("\r\r\r%d", $index);

				$zipPath = $gallery->getArchiveFilepath();
				if(!file_exists($zipPath)) {
					Log::error(self::LOG_TAG, 'Zip file doesn\'t exist: %s', $zipPath);
				}

				$convertedPath = preg_replace('/.zip$/', '.tar', $zipPath);

				if(file_exists($convertedPath)) {
					unlink($convertedPath);
				}

				$zip = new PharData($zipPath);
				$tar = $zip->convertToData(Phar::TAR);
				unset($tar);

				$tarFolder = sprintf('%s/archives/%s', Config::get()->archiveDir, $gallery->getRoundedId());
				if(!file_exists($tarFolder)) {
					mkdir($tarFolder);
				}

				$tarPath = sprintf('%s/%d.tar', $tarFolder, $gallery->exhenid);
				rename($convertedPath, $tarPath);

				$index++;

				exit;
			}

			$page++;
		}
	}
}

?>


text:
<?php

class Task_Audit extends Task_Abstract {

    const LOG_TAG = 'Task_Audit';

    protected $client;

    public function run($options = array()) {
        $this->client = new ExClient();

        while(true) {

            $interval = isset($options[0]) ? $options[0] : "1 year";

            $galleries = R::find('gallery',
                'archived = 1 and deleted = 0 and source = 0 and'.
                '((added <= date_sub(date(now()), interval 3 day) and'. // if added more than 3 days ago...
                '(posted >= date_sub(date(now()), interval ' . $interval . ') or (lastaudit is null)) and'. // and if the gallery is more than input old but NOT if it's unaudited
                '(lastaudit is null or lastaudit <= date_sub(date(now()), interval 7 day))) or'. // ...and not yet audited, or audited more than 7 days ago
				'((added >= date_sub(date(now()), interval 7 day) and added <= date_sub(date(now()), interval 1 day)) and'. // OR, added less than 7 days ago (but more than 24 hours ago)...
                '(lastaudit is null or lastaudit <= date_sub(date(now()), interval 1 day))))'. // ...and not yet audited, or audited more than 1 day ago
                'order by posted desc limit 100');

            if(count($galleries) === 0) {
                break;
            }

            foreach($galleries as $gallery) {
                $this->audit($gallery);
            }
        }

        if(isset(Config::get()->indexer->full)) {
            $command = Config::get()->indexer->full;
            system($command);
        }
    }

    protected function audit($gallery) {
        Log::debug(self::LOG_TAG, 'Auditing gallery: #%d - %s', $gallery->exhenid, $gallery->name);

        $galleryHtml = $this->client->gallery($gallery->exhenid, $gallery->hash);

        // Galleries that are removed now return a 404, so we have to leave them as audited completely for now.
        if(!$galleryHtml) {
            $gallery->lastaudit = date('Y-m-d H:i:s'); //gallery was probably deleted, so mark it as audited for now
            R::store($gallery);

            Log::error(self::LOG_TAG, 'Gallery was either removed or was invalid.');
            return;
        }

        $galleryPage = new ExPage_Gallery($galleryHtml);
        if(!$galleryPage->isValid()) {
            $gallery->lastaudit = date('Y-m-d H:i:s'); //gallery was probably deleted, so mark it as audited for now
            R::store($gallery);

            Log::error(self::LOG_TAG, 'Gallery was either removed or was invalid.');
            return;
        }

        $childGallery = $galleryPage->getNewestVersion();
        if($childGallery) {

            $childHtml = $this->client->gallery($childGallery->exhenid, $childGallery->hash);
            $childPage = new ExPage_Gallery($childHtml);

            if ($childPage->isValid()) {
                Log::debug(self::LOG_TAG, 'New gallery found for gallery (%d): #%d - %s', $childGallery->exhenid, $gallery->exhenid, $gallery->name);

                Model_Gallery::addGallery($childGallery->exhenid, $childGallery->hash);

                $gallery->deleted = 1;
            }

        }
        else {
            $newTags = $galleryPage->getTags();
            $oldTags = $gallery->exportTags();

            if(count($newTags) > 0) {
                $diff = self::tagsDiff($oldTags, $newTags);

                if(count($diff) > 0) {
                    $humanDiff = array();
                    foreach($diff as $ns => $tags) {
                        foreach($tags as $tag) {
                            $humanDiff[] = $ns.':'.$tag;
                        }
                    }

                    $humanDiff = implode(', ', $humanDiff);

                    Log::debug(self::LOG_TAG, 'Different tags found for gallery: #%d - %s (%s)', $gallery->exhenid, $gallery->name, $humanDiff);
                    $gallery->ownGalleryTag = array();
                    $gallery->addTags($newTags);

                    $cache = Cache::getInstance();
                    $cache->deleteObject('gallery', $gallery->id);
                }
            }
        }

        $gallery->lastaudit = date('Y-m-d H:i:s');

        if ($gallery->deleted == 1) {
            $gallery->lastaudit = null;
        }

        R::store($gallery);
    }

    static function tagsDiff($tags1, $tags2){
        $ret = array();

        foreach ($tags1 as $ns => $tags) {
            if(!array_key_exists($ns, $tags2)) {
                $ret[$ns] = $tags;
                continue;
            }

            foreach($tags as $index => $tag) {
                if(!in_array($tag, $tags2[$ns])) {
                    if(!array_key_exists($ns, $ret) || !is_array($ret[$ns])) {
                        $ret[$ns] = array();
                    }

                    $ret[$ns][] = $tag;
                }
                else {
                    $pos = array_search($tag, $tags2[$ns]);
                    unset($tags2[$ns][$pos]);
                }
            }

            if(count($tags2[$ns]) > 0) {
                if(!array_key_exists($ns, $ret) || !is_array($ret[$ns])) {
                    $ret[$ns] = array();
                }

                $ret[$ns] = array_merge($ret[$ns], $tags2[$ns]);
            }

            unset($tags2[$ns]);
        }

        $ret = array_merge($ret, $tags2);

        return $ret;
    }

}

?>


text:
<?php

class Task_Cleanup extends Task_Abstract {

	const LOG_TAG = 'Task_Cleanup';

	public function run($options = array()) {
		$config = Config::get();
        $cache = Cache::getInstance();

		Log::debug(self::LOG_TAG, 'Checking for NULL galleryproperty entries');

		$entries = R::getAll('SELECT * FROM galleryproperty WHERE gallery_id IS NULL');
		if (count($entries) > 0) {
			$count = 0;
			Log::debug(self::LOG_TAG, 'NULL entries found, deleting them now.');
			foreach($entries as $entry) {
				$book = R::load('galleryproperty', $entry["id"]);
				R::trash($book);
				$count++;
			}
			Log::debug(self::LOG_TAG, 'All NULL entries deleted. Total deleted was %d', $count);
		}
		else {
			Log::debug(self::LOG_TAG, 'No NULL galleryproperty entries found.');
		}

        Log::debug(self::LOG_TAG, 'Checking for NULL gallery_tag entries');
        $entries = R::getAll('SELECT * FROM gallery_tag WHERE gallery_id IS NULL');
        if (count($entries) > 0) {
            $count = 0;
            Log::debug(self::LOG_TAG, 'NULL entries found, deleting them now.');
            foreach($entries as $entry) {
                $book = R::load('gallery_tag', $entry["id"]);
                R::trash($book);
                $count++;
            }
            Log::debug(self::LOG_TAG, 'All NULL entries deleted. Total deleted was %d', $count);
        } else {
            Log::debug(self::LOG_TAG, 'No NULL gallery_tag entries found.');
        }

		Log::debug(self::LOG_TAG, 'Checking for any galleries marked as deleted.');

		$entries = R::getAll('SELECT * FROM gallery WHERE deleted = 1');
		if (count($entries) > 0) {
			$count = 0;
			Log::debug(Self::LOG_TAG, 'Deleting galleries now. This may take a while.');
			foreach ($entries as $entry) {
				$gallery = R::load('gallery', $entry["id"]);

                Log::debug(Self::LOG_TAG, 'Deleting Gallery ' . $entry["name"] . ' (' . $entry["id"] . ')');
                $galleryproperty = R::getAll('SELECT * FROM galleryproperty WHERE gallery_id = ?', array($entry["id"]));
                $gallery_tag = R::getAll('SELECT * FROM gallery_tag WHERE gallery_id = ?', array($entry["id"]));
                $gallery_thumb = R::getAll('SELECT * FROM gallery_thumb WHERE gallery_id = ?', array($entry["id"]));

                // Get all image entries associated with the gallery_thumb then delete those, then delete the gallery_thumb entry
                foreach ($gallery_thumb as $thumb) {
                    $images = R::getAll('SELECT * FROM image WHERE id = ?', array($thumb["image_id"]));
                    // Delete all associated images and the thumbnails
                    foreach ($images as $image) {
                        $book = R::load('image', $image["id"]);
                        $imageName = str_pad($book["id"], 6, '0', STR_PAD_LEFT) . '.jpeg';
                        unlink($config->imagesDir . '/' . $imageName);
                        R::trash($book);
                    }
                    $book = R::load('gallery_thumb', $thumb["id"]);
                    R::trash($book);
                }

                // Delete all associated tags (just the references, not the actual tag entries)
                foreach ($gallery_tag as $tag) {
                    $book = R::load("gallery_tag", $tag["id"]);
                    R::trash($book);
                }

                // Delete all associated properties.
                foreach ($galleryproperty as $prop) {
                    $book = R::load('galleryproperty', $prop["id"]);
                    R::trash($book);
                }

                // Delete archive
                $archiveName = $gallery["exhenid"] . '.zip';
                unlink($config->archiveDir . '/galleries/' . $archiveName);

                // Delete html file
                $htmlName = $gallery["exhenid"] . '.html';
                unlink($config->archiveDir . '/pages/' . $htmlName);

                if ($gallery["feed_id"] == null) {
                    R::trash($gallery);
                } else {
                    $gallery["deleted"] = 2;
                    R::store($gallery);
                }

                $cache->deleteObject('gallery', $gallery->id);
                $count++;
			}
			Log::debug(self::LOG_TAG, 'All galleries marked as deleted were deleted. Total deleted was %d', $count);
            if(isset($config->indexer->full)) {
                $command = Config::get()->indexer->full;
                system($command);
            }
		}
		else {
			Log::debug(self::LOG_TAG, 'No galleries found that were marked as deleted.');
		}
	}

}

?>


text:
<?php

class Task_EditGallery extends Task_Abstract {

	const LOG_TAG = 'Task_EditGallery';

	public function run($options = array()) {

		if(count($options) < 2) {
			printf("Invalid options.\n");
			printf("Example: TaskRunner EditGallery id action params...\n");
			printf("Valid actions:\n");
			printf("\taddtag namespace:tag ...\n");
			printf("\n");
			return;
		}

		$galleryId = $options[0];
		$gallery = R::load('gallery', $galleryId);
		if(!$gallery->id) {
			Log::error(self::LOG_TAG, 'Failed to load gallery with id: %s', $galleryId);
			return;
		}

		$action = $options[1];
		$params = array_slice($options, 2);

		if($action === 'addtag') {
			if(count($params) < 1) {
				Log::error(self::LOG_TAG, 'Missing tag to add');
				return;
			}

			foreach($params as $tagParam) {
				$bits = explode(':', $tagParam);
				if(count($bits) !== 2) {
					Log::error(self::LOG_TAG, 'Missing tag namespace: %s', $tagParam);
					continue;
				}

				list($ns, $tag) = $bits;

				if($gallery->hasTag($ns, $tag)) {
					Log::error(self::LOG_TAG, 'Gallery already has tag: %s:%s', $ns, $tag);
					continue;
				}
				else {
					$gallery->addTags(array($ns => array($tag)));
				}
			}
		}
		else {
			Log::error(self::LOG_TAG, 'Invalid action: %s', $action);
			return;
		}

		R::store($gallery);

		$cache = Cache::getInstance();
		$cache->deleteObject('gallery', $gallery->id);

		if(isset(Config::get()->indexer->full)) {
			$command = Config::get()->indexer->full;
    		system($command);
    	}
	}

}

?>


text:
<?php

class Task_AddGallery extends Task_Abstract {

	const LOG_TAG = 'Task_AddGallery';

	public function run($options = array()) {

		if(count($options) !== 2) {
			printf("Invalid options.\n");
			printf("Example: TaskRunner AddGallery input.zip galleryname\n\n");
			return;
		}

		list($inputFile, $galleryName) = $options;

		if(!file_exists($inputFile)) {
			Log::error(self::LOG_TAG, 'Failed to open input file');
			return;
		}

		$gallery = R::dispense('gallery');
		$gallery->name = $galleryName;
		$gallery->archived = 1;
		$gallery->added = date('Y-m-d H:i:s');
		$gallery->posted = date('Y-m-d H:i:s');
		$gallery->source = Model_Gallery::SOURCE_MANUAL;

		$archive = new ZipArchive();
		$ret = $archive->open($inputFile);
		if($ret === true && $archive->status == ZipArchive::ER_OK) {
			$gallery->numfiles = $archive->numFiles;
			$archive->close();

			$gallery->filesize = filesize($inputFile);
			$gallery->archived = true;
		}
		else {
			Log::error(self::LOG_TAG, 'Input file is not a valid zip');
		}

		R::store($gallery);

		$dest = $gallery->getArchiveFilepath();
		$ret = copy($inputFile, $dest);

		if(!$ret) {
			Log::error(self::LOG_TAG, 'Failed to copy file to dest: %s', $dest);
			R::trash($gallery);
			return;
		}

		if(isset(Config::get()->indexer->full)) {
			$command = Config::get()->indexer->full;
    		system($command);
    	}
	}

}

?>


text:
<?php

class Task_ForceAudit extends Task_Abstract {

    const LOG_TAG = 'Task_ForceAudit';

    protected $client;

    public function run($options = array()) {
        $this->client = new ExClient();

        if (isset($options[0])) {
            $mode = strtolower($options[0]);
        } else {
            exit;
        }

        if ($mode == "all") {
            $galleries = R::findAll('gallery',
                'archived = 1 and deleted = 0 and source = 0');
        }

        if ($mode == "id") {
            if (isset($options[1])) {
                $id = (int)$options[1];
                $galleries = R::findAll('gallery',
                    'archived = 1 and deleted = 0 and source = 0 and id = ?',
                    [$id]);
            } else {
                exit;
            }
        }

        if ($mode == "before") {
            if (isset($options[1])) {
                $date = strtotime($options[1]);
                $date = date("Y-m-d H:i:s", $date);
                $galleries = R::findAll('gallery',
                    'archived = 1 and deleted = 0 and source = 0 ' .
                    'and added <= ?', [$date]);
            } else {
                exit;
            }
        }

        if ($mode == "after") {
            if (isset($options[1])) {
                $date = strtotime($options[1]);
                $date = date("Y-m-d H:i:s", $date);
                $galleries = R::findAll('gallery',
                    'archived = 1 and deleted = 0 and source = 0 ' .
                    'and added >= ?', [$date]);
            } else {
                exit;
            }
        }

        if ($mode == "range" || $mode == "between") {
            if (isset($options[1]) && isset($options[2])) {
                $startDate = strtotime($options[1]);
                $startDate = date("Y-m-d H:i:s", $startDate);

                $endDate = strtotime($options[2]);
                $endDate = date("Y-m-d H:i:s", $endDate);

                $galleries = R::findAll('gallery',
                    'archived = 1 and deleted = 0 and source = 0 ' .
                    'and added between :start and :end',
                    array(":start" => $startDate, ":end" => $endDate));
            } else {
                exit;
            }
        }

        if (count($galleries) == 0) {
            exit;
        }

        foreach($galleries as $gallery) {
            $this->audit($gallery);
        }

        if(isset(Config::get()->indexer->full)) {
            $command = Config::get()->indexer->full;
            system($command);
        }
    }

    protected function audit($gallery) {
        Log::debug(self::LOG_TAG, 'Auditing gallery: #%d - %s', $gallery->exhenid, $gallery->name);

        $galleryHtml = $this->client->gallery($gallery->exhenid, $gallery->hash);

        // Galleries that are removed now return a 404, so we have to leave them as audited completely for now.
        if(!$galleryHtml) {
            $gallery->lastaudit = date('Y-m-d H:i:s'); //gallery was probably deleted, so mark it as audited for now
            R::store($gallery);

            Log::error(self::LOG_TAG, 'Gallery was either removed or was invalid.');
            return;
        }

        $galleryPage = new ExPage_Gallery($galleryHtml);
        if(!$galleryPage->isValid()) {
            $gallery->lastaudit = date('Y-m-d H:i:s'); //gallery was probably deleted, so mark it as audited for now
            R::store($gallery);

            Log::error(self::LOG_TAG, 'Gallery was either removed or was invalid.');
            return;
        }

        $childGallery = $galleryPage->getNewestVersion();
        if($childGallery) {

            $childHtml = $this->client->gallery($childGallery->exhenid, $childGallery->hash);
            $childPage = new ExPage_Gallery($childHtml);

            if ($childPage->isValid()) {
                Log::debug(self::LOG_TAG, 'New gallery found for gallery (%d): #%d - %s', $childGallery->exhenid, $gallery->exhenid, $gallery->name);

                Model_Gallery::addGallery($childGallery->exhenid, $childGallery->hash);

                $gallery->deleted = 1;
            }

        }
        else {
            $newTags = $galleryPage->getTags();
            $oldTags = $gallery->exportTags();

            if(count($newTags) > 0) {
                $diff = self::tagsDiff($oldTags, $newTags);

                if(count($diff) > 0) {
                    $humanDiff = array();
                    foreach($diff as $ns => $tags) {
                        foreach($tags as $tag) {
                            $humanDiff[] = $ns.':'.$tag;
                        }
                    }

                    $humanDiff = implode(', ', $humanDiff);

                    Log::debug(self::LOG_TAG, 'Different tags found for gallery: #%d - %s (%s)', $gallery->exhenid, $gallery->name, $humanDiff);
                    $gallery->ownGalleryTag = array();
                    $gallery->addTags($newTags);

                    $cache = Cache::getInstance();
                    $cache->deleteObject('gallery', $gallery->id);
                }
            }
        }

        $gallery->lastaudit = date('Y-m-d H:i:s');

        if ($gallery->deleted == 1) {
            $gallery->lastaudit = null;
        }

        R::store($gallery);
    }

    static function tagsDiff($tags1, $tags2){
        $ret = array();

        foreach ($tags1 as $ns => $tags) {
            if(!array_key_exists($ns, $tags2)) {
                $ret[$ns] = $tags;
                continue;
            }

            foreach($tags as $index => $tag) {
                if(!in_array($tag, $tags2[$ns])) {
                    if(!array_key_exists($ns, $ret) || !is_array($ret[$ns])) {
                        $ret[$ns] = array();
                    }

                    $ret[$ns][] = $tag;
                }
                else {
                    $pos = array_search($tag, $tags2[$ns]);
                    unset($tags2[$ns][$pos]);
                }
            }

            if(count($tags2[$ns]) > 0) {
                if(!array_key_exists($ns, $ret) || !is_array($ret[$ns])) {
                    $ret[$ns] = array();
                }

                $ret[$ns] = array_merge($ret[$ns], $tags2[$ns]);
            }

            unset($tags2[$ns]);
        }

        $ret = array_merge($ret, $tags2);

        return $ret;
    }

}

?>


text:
<?php

class Task_Archive extends Task_Abstract {

	public function run($options = array()) {
		$archiver = new ExArchiver();
		if (isset($options[0])) {
			$opt = strtolower($options[0]);

			if ($opt == "feed") {
				$archiver->start((int)$options[1]);
			} else {
				$archiver->start();
			}
		} else {
			$archiver->start();
		}

	}

}

?>


text:
<?php

abstract class Task_Abstract {
	public abstract function run($options = array());
}

?>


text:
<?php

class Task_Thumbnails extends Task_Abstract {

	const LOG_TAG = 'Task_Thumbnails';

	public function run($options = array()) {
		$galleries = R::find('gallery', 'numfiles != (SELECT count(*) FROM exhen.gallery_thumb where gallery_thumb.type = 2 and gallery_id = gallery.id)');

		$count = 0;

		foreach($galleries as $gallery) {
			$success = true;

        	for($i = 0; $i < $gallery->numfiles; $i++) {
                try {
                	$gallery->getThumbnail($i, 2, true);
				}
                catch(Exception $e) {
					$success = false;
                }
        	}

			if($success) {
				$count++;
			}
		}

		Log::debug(self::LOG_TAG, 'Processed thumbnails for %d galleries', $count);
	}

}

?>


text:
<?php

use PHPImageWorkshop\ImageWorkshop;

class Model_Gallery extends Model_Abstract
{
    const SOURCE_EXHENTAI = 0;
    const SOURCE_MANUAL = 1;
    const SOURCE_NHENTAI = 2;

    const THUMB_LARGE = 1;
    const THUMB_SMALL = 2;

    const GP_PER_MB = 41.9435018158;

    public static function search($page, $pagesize, $search, $order, $randomSeed = null, $unarchived = false, $read = false, $color = false)
    {
        $query = new QueryHelper();

        if ($order === 'random') {
            $query->sql('select *, crc32(to_string(id + :seed)) as rnd from galleries');
            $query->addParams(array('seed' => $randomSeed));
        } elseif ($order === 'weight') {
            $query->sql('select *, weight() as ranked_weight from galleries');
        } else {
            $query->sql('select * from galleries');
        }

        $query->sql('where deleted = 0');

        if (!$unarchived) {
            $query->sql('and archived = 1');
        }

        if ($read !== false) {
            $query->sql('and read = :read')
                ->addParams(array('read' => $read));
        }

        if ($color !== false) {
            $query->sql('and color = :color')
                ->addParams(array('color' => $color));
        }

        if ($search) {
            $search = SphinxQL::halfEscapeMatch($search);

            $query->sql('and match(:match)')
                ->addParams(array('match' =>  $search));
        }

        if ($order === 'added') {
            $query->sql('order by added desc');
        } elseif ($order === 'random') {
            $query->sql('order by rnd asc');
        } elseif ($order === 'weight') {
            $query->sql('order by ranked_weight desc, posted desc');
        } else {
            $query->sql('order by posted desc');
        }

        if ($page >= 0 && $pagesize) {
            $query->sql('limit :offset, :limit')
                ->addParams(array(
                    'offset' => (int)($page * $pagesize),
                    'limit' => (int)$pagesize
                ));
        }

        if ($order === 'weight') {
            $query->sql('option ranker = wordcount,');
        } else {
            $query->sql('option ranker = none,');
        }

        $query->sql('max_matches = 500000');

        $result = SphinxQL::query($query->getSql(), $query->getParams());
        $meta = SphinxQL::getMeta();

        return array('result' => $result, 'meta' => $meta);
    }

    // this really needs redoing..
    public function getArchiveFilepath()
    {
        if ($this->source == self::SOURCE_EXHENTAI) {
            return sprintf('%s/galleries/%d.zip', Config::get()->archiveDir, $this->exhenid);
        } else {
            return sprintf('%s/galleries/%d-%d.zip', Config::get()->archiveDir, $this->source, $this->id);
        }
    }

    public function getImageFilepath($index)
    {
        $zipPath = $this->getArchiveFilepath();
        if (file_exists($zipPath)) {
            $files = array();

            $zip = new PharData($zipPath);
            if ($zip) {
                foreach ($zip as $file) {
                    $files[] = basename($file);
                }

                natcasesort($files);
                $files = array_values($files); //strip keys

                if (array_key_exists($index, $files)) {
                    return sprintf('phar://%s/%s', $zipPath, $files[$index]);
                }
            }
        }

        return false;
    }

    public function getImageBean($index)
    {
        return R::findOne('galleryimage', 'gallery_id = ? and galleryimage.index = ?', array($this->id, $index));
    }

    public function getThumbnail($index, $type, $create = true)
    {
        if ($this->archived) {
            $link = $this->unbox()->withCondition('gallery_thumb.index = ? and gallery_thumb.type = ?', array($index, $type))->via('gallery_thumb')->ownGalleryThumb;

            if (count($link) > 0) {
                $link = array_pop($link);

                return $link->image;
            } elseif ($create) {
                $inFile = $this->getImageFilepath($index);

                if (file_exists($inFile)) {
                    $resizedFilename = sprintf('resized_gallery_%d_%d.jpg', $this->id, $index);

                    $layer = ImageWorkshop::initFromPath($inFile);

                    if ($type == self::THUMB_LARGE) {
                        $layer->resizeInPixel(350, null, true);
                    } elseif ($type == self::THUMB_SMALL) {
                        $layer->resizeInPixel(140, null, true);
                    } else {
                        return false;
                    }

                    $tempDir = Config::get()->tempDir;
                    $layer->save($tempDir, $resizedFilename, true, null, 95);

                    $outFile = $tempDir.DS.$resizedFilename;
                    $image = Model_Image::importFromFile($outFile);

                    unlink($outFile);

                    $link = $image->unbox()->link('gallery_thumb', array('index' => $index, 'type' => $type))->gallery = $this->unbox();
                    R::store($image);

                    return $image;
                }
            }
        }

        return false;
    }

    public function dispense()
    {
        $this->added = date('Y-m-d H:i:s');
    }

    public function update()
    {
        $this->updated = date('Y-m-d H:i:s');
    }

    public function getRoundedId()
    {
        return round(($this->exhenid - 500), -4);
    }

    public static function getStats()
    {
        $sql =
            'select
				count(id) as count_total,
				sum(filesize) as filesize,
				round(sum(filesize) / pow(1024, 3), 2) as filesize_gb,
				(select count(id) from gallery where archived = 1) as count_archived,
				(select count(id) from gallery where deleted >= 1) as count_deleted,
				round((sum(filesize) / (select count(id) from gallery where archived = 1)) / pow(1024, 2), 2) as filesize_average_mb,
				sum(round(round(filesize / pow(1024, 2), 2) * 41.9435018158)) as gp_total,
				round(sum(round(filesize / pow(1024, 2), 2) * 41.9435018158) / (select count(id) from gallery where archived = 1)) as gp_average
			from gallery
			limit 1';

        $stats = R::getRow($sql);

        return $stats;
    }

    public function hasTag($ns, $tag)
    {
        $tagBean = R::findOne('tag', 'name = ?', array($tag));
        $nsBean = R::findOne('tagnamespace', 'name = ?', array($ns));

        if (!$tagBean || !$nsBean) {
            return false;
        }

        $result = $this->unbox()->withCondition('tag_id = ? and namespace_id = ?', array($tagBean->id, $nsBean->id))->ownGalleryTag;
        return (count($result) > 0);
    }

    public function addTags($tagList)
    {
        foreach ($tagList as $tagNamespace => $tags) {
            $ns = R::findOne('tagnamespace', 'name = ?', array($tagNamespace));
            if (!$ns) {
                $ns = R::dispense('tagnamespace');
                $ns->name = $tagNamespace;
                R::store($ns);
            }

            foreach ($tags as $tagName) {
                $tag = R::findOne('tag', 'name = ?', array($tagName));
                if (!$tag) {
                    $tag = R::dispense('tag');
                    $tag->name = $tagName;
                    R::store($tag);
                }

                $this->unbox()->link('gallery_tag', array('namespace' => $ns))->tag = $tag;
            }
        }
    }

    public function addProperties($props)
    {
        foreach ($props as $name => $value) {
            $galleryAttr = R::dispense('galleryproperty');
            $galleryAttr->name = $name;
            $galleryAttr->value = $value;
            $galleryAttr->gallery = $this->unbox();
            R::store($galleryAttr);

            if ($name === 'Posted') {
                $this->unbox()->posted = $value;
            }
        }
    }

    public static function addGallery($gid, $hash)
    {
        $gallery = R::findOne('gallery', 'exhenid = ?', array($gid));
        if (!$gallery) {
            $gallery = R::dispense('gallery');
            $gallery->exhenid = $gid;
            $gallery->hash = $hash;
            R::store($gallery);
        } elseif (!$gallery->download) {
            $gallery->download = true;
            $gallery->added = date('Y-m-d H:i:s');
            R::store($gallery);
        }

        return $gallery;
    }

    public function exportTags()
    {
        $tagLinks = $this->ownGalleryTag;

        R::preload($tagLinks, array('namespace' => 'tagnamespace'));

        $export = array();
        foreach ($tagLinks as $link) {
            if (!array_key_exists($link->namespace->name, $export)) {
                $export[$link->namespace->name] = array();
            }

            $export[$link->namespace->name][] = $link->tag->name;
        }

        return $export;
    }

    public function exportSource()
    {
        return $this->source;
    }

    public function exportFeed()
    {
        return $this->feed_id;
    }
}


text:
<?php

class Model_Image extends Model_Abstract
{
    public function delete()
    {
        $filepath = $this->getFilepath();
        if (file_exists($filepath)) {
            unlink($filepath);
        }
    }

    public function getFilepath()
    {
        if ($this->id) {
            return realpath(Config::get()->imagesDir).DS.str_pad($this->id, 6, '0', STR_PAD_LEFT).'.'.$this->type;
        }
    }

    public function getUrl()
    {
        if ($this->id) {
            return '/images/'.str_pad($this->id, 6, '0', STR_PAD_LEFT).'.'.$this->type;
        }
    }

    public static function importFromFile($filepath)
    {
        if (!file_exists($filepath)) {
            throw new Exception('Input file does not exist');
        }

        $ret = getimagesize($filepath);
        if (!$ret) {
            throw new Exception('Input file is invalid');
        }

        list($width, $height, $type, $attr) = $ret;

        $image = R::dispense('image');
        R::store($image);
        $image->width = $width;
        $image->height = $height;
        $image->type = image_type_to_extension($type, false);
        $image->filename = pathinfo($filepath, PATHINFO_BASENAME);
        R::store($image);

        $ret = copy($filepath, $image->getFilepath());
        if (!$ret) {
            R::trash($image);
            throw new Exception('Failed to copy image');
        }

        return $image;
    }
}


text:
<?php

abstract class Model_Abstract extends RedBean_SimpleModel
{

}