text:
#!/usr/bin/env python3

# This implementation is modified from
# https://github.com/alexdej/puzpy/
# Removes py2 support, and adds the ability to add
# rebus clues to a new file

import functools
import operator
import math
import string
import struct

__title__ = 'puzpy'
__version__ = '0.2.3'
__author__ = 'Alex DeJarnatt'
__author_email__ = 'adejarnatt@gmail.com'
__maintainer__ = 'Simeon Visser'
__maintainer_email__ = 'simeonvisser@gmail.com'
__license__ = 'MIT'
__copyright__ = 'Copyright 2009 Alex DeJarnatt'

HEADER_FORMAT = '''<
             H 11s        xH
             Q       4s  2sH
             12s         BBH
             H H '''

HEADER_CKSUM_FORMAT = '<BBH H H '

EXTENSION_HEADER_FORMAT = '< 4s  H H '

MASKSTRING = 'ICHEATED'

ENCODING = 'ISO-8859-1'
ENCODING_UTF8 = 'UTF-8'
ENCODING_ERRORS = 'namereplace'  # e.g. '\N{EM DASH}'. Least-worst option.

ACROSSDOWN = b'ACROSS&DOWN'

BLACKSQUARE = '.'
BLACKSQUARE2 = ':'

def enum(**enums):
    return type('Enum', (), enums)

PuzzleType = enum(
    Normal=0x0001,
    Diagramless=0x0401
)

# the following diverges from the documentation
# but works for the files I've tested
SolutionState = enum(
    # solution is available in plaintext
    Unlocked=0x0000,
    # solution is locked (scrambled) with a key
    Locked=0x0004
)

GridMarkup = enum(
    # ordinary grid cell
    Default=0x00,
    # marked incorrect at some point
    PreviouslyIncorrect=0x10,
    # currently showing incorrect
    Incorrect=0x20,
    # user got a hint
    Revealed=0x40,
    # circled
    Circled=0x80
)

# refer to Extensions as Extensions.Rebus, Extensions.Markup
Extensions = enum(
    # grid of rebus indices: 0 for non-rebus;
    # i+1 for key i into RebusSolutions map
    Rebus=b'GRBS',
    # map of rebus solution entries eg 0:HEART;1:DIAMOND;17:CLUB;23:SPADE;
    RebusSolutions=b'RTBL',
    # user's rebus entries
    RebusFill=b'RUSR',
    # timer state: 'a,b' where a is the number of seconds elapsed and
    # b is a boolean (0,1) for whether the timer is running
    Timer=b'LTIM',
    # grid cell markup: previously incorrect: 0x10;
    # currently incorrect: 0x20,
    # hinted: 0x40,
    # circled: 0x80
    Markup=b'GEXT'
)

def read(filename):
    """
    Read a .puz file and return the Puzzle object.
    throws PuzzleFormatError if there's any problem with the file format.
    """
    with open(filename, 'rb') as f:
        return load(f.read())

def load(data):
    """
    Read .puz file data and return the Puzzle object.
    throws PuzzleFormatError if there's any problem with the file format.
    """
    puz = Puzzle()
    puz.load(data)
    return puz

class PuzzleFormatError(Exception):
    """
    Indicates a format error in the .puz file. May be thrown due to
    invalid headers, invalid checksum validation, or other format issues.
    """
    def __init__(self, message=''):
        self.message = message

class Puzzle:
    """Represents a puzzle
    """
    def __init__(self):
        """Initializes a blank puzzle
        """
        self.preamble = b''
        self.postscript = b''
        self.title = ''
        self.author = ''
        self.copyright = ''
        self.width = 0
        self.height = 0
        self.version = b'1.3'
        self.fileversion = b'1.3\0'  # default
        self.encoding = ENCODING
        # these are bytes that might be unused
        self.unk1 = b'\0' * 2
        self.unk2 = b'\0' * 12
        self.scrambled_cksum = 0
        self.fill = ''
        self.solution = ''
        self.clues = []
        self.notes = ''
        self.extensions = {}
        # the folowing is so that we can round-trip values in order:
        self._extensions_order = []
        self.puzzletype = PuzzleType.Normal
        self.solution_state = SolutionState.Unlocked
        self.helpers = {}  # add-ons like Rebus and Markup

    def load(self, data):
        s = PuzzleBuffer(data)

        # advance to start - files may contain some data before the
        # start of the puzzle use the ACROSS&DOWN magic string as a waypoint
        # save the preamble for round-tripping
        if not s.seek_to(ACROSSDOWN, -2):
            raise PuzzleFormatError("Data does not appear to represent a "
                                    "puzzle. Are you sure you didn't intend "
                                    "to use read?")

        self.preamble = s.data[:s.pos]

        puzzle_data = s.unpack(HEADER_FORMAT)
        cksum_gbl = puzzle_data[0]
        # acrossDown = puzzle_data[1]
        cksum_hdr = puzzle_data[2]
        cksum_magic = puzzle_data[3]
        self.fileversion = puzzle_data[4]
        # since we don't know the role of these bytes, just round-trip them
        self.unk1 = puzzle_data[5]
        self.scrambled_cksum = puzzle_data[6]
        self.unk2 = puzzle_data[7]
        self.width = puzzle_data[8]
        self.height = puzzle_data[9]
        numclues = puzzle_data[10]
        self.puzzletype = puzzle_data[11]
        self.solution_state = puzzle_data[12]

        self.version = self.fileversion[:3]
        # Once we have fileversion we can guess the encoding
        self.encoding = ENCODING if self.version_tuple()[0] < 2 else ENCODING_UTF8
        s.encoding = self.encoding

        self.solution = s.read(self.width * self.height).decode(self.encoding)
        self.fill = s.read(self.width * self.height).decode(self.encoding)

        self.title = s.read_string()
        self.author = s.read_string()
        self.copyright = s.read_string()

        self.clues = [s.read_string() for i in range(0, numclues)]
        self.notes = s.read_string()

        ext_cksum = {}
        while s.can_unpack(EXTENSION_HEADER_FORMAT):
            code, length, cksum = s.unpack(EXTENSION_HEADER_FORMAT)
            ext_cksum[code] = cksum
            # extension data is represented as a null-terminated string,
            # but since the data can contain nulls we can't use read_string
            self.extensions[code] = s.read(length)
            s.read(1)  # extensions have a trailing byte
            # save the codes in order for round-tripping
            self._extensions_order.append(code)

        # sometimes there's some extra garbage at
        # the end of the file, usually \r\n
        if s.can_read():
            self.postscript = s.read_to_end()

        if cksum_gbl != self.global_cksum():
            raise PuzzleFormatError('global checksum does not match')
        if cksum_hdr != self.header_cksum():
            raise PuzzleFormatError('header checksum does not match')
        if cksum_magic != self.magic_cksum():
            raise PuzzleFormatError('magic checksum does not match')
        for code, cksum_ext in ext_cksum.items():
            if cksum_ext != data_cksum(self.extensions[code]):
                raise PuzzleFormatError(
                    'extension %s checksum does not match' % code
                )

    def save(self, filename):
        puzzle_bytes = self.tobytes()
        with open(filename, 'wb') as f:
            f.write(puzzle_bytes)

    def tobytes(self):
        s = PuzzleBuffer(encoding=self.encoding)
        # commit any changes from helpers
        for h in self.helpers.values():
            if 'save' in dir(h):
                h.save()

        # include any preamble text we might have found on read
        s.write(self.preamble)

        s.pack(HEADER_FORMAT,
               self.global_cksum(), ACROSSDOWN,
               self.header_cksum(), self.magic_cksum(),
               self.fileversion, self.unk1, self.scrambled_cksum,
               self.unk2, self.width, self.height,
               len(self.clues), self.puzzletype, self.solution_state)

        s.write(self.encode(self.solution))
        s.write(self.encode(self.fill))

        s.write_string(self.title)
        s.write_string(self.author)
        s.write_string(self.copyright)

        for clue in self.clues:
            s.write_string(clue)

        s.write_string(self.notes)

        # do a bit of extra work here to ensure extensions round-trip in the
        # order they were read. this makes verification easier. But allow
        # for the possibility that extensions were added or removed from
        # self.extensions
        ext = dict(self.extensions)
        for code in self._extensions_order:
            data = ext.pop(code, None)
            if data:
                s.pack(EXTENSION_HEADER_FORMAT, code,
                       len(data), data_cksum(data))
                s.write(data + b'\0')

        for code, data in ext.items():
            s.pack(EXTENSION_HEADER_FORMAT, code, len(data), data_cksum(data))
            s.write(data + b'\0')

        # postscript is initialized, read, and stored as bytes. In case it is
        # overwritten as a string, this try/except converts it back.
        try:
            postscript_bytes = self.encode(self.postscript)
        except AttributeError:
            postscript_bytes = self.postscript

        s.write(postscript_bytes)

        return s.tobytes()

    def encode(self, s):
        return s.encode(self.encoding, ENCODING_ERRORS)

    def encode_zstring(self, s):
        return self.encode(s) + b'\0'

    def version_tuple(self):
        return tuple(map(int, self.version.split(b'.')))

    def has_rebus(self):
        return self.rebus().has_rebus()

    def create_empty_rebus(self):
        rebus = self.rebus()
        self.extensions[Extensions.Rebus] = b''
        self.extensions[Extensions.RebusSolutions] = b''
        rebus.fill = {}
        rebus.solutions = {}
        rebus.table = []
        return rebus

    def rebus(self):
        return self.helpers.setdefault('rebus', Rebus(self))

    def has_markup(self):
        return self.markup().has_markup()

    def markup(self):
        return self.helpers.setdefault('markup', Markup(self))

    def clue_numbering(self):
        numbering = DefaultClueNumbering(self.fill, self.clues, self.width, self.height)
        return self.helpers.setdefault('clues', numbering)

    def blacksquare(self):
        return BLACKSQUARE2 if self.puzzletype == PuzzleType.Diagramless else BLACKSQUARE

    def is_solution_locked(self):
        return bool(self.solution_state != SolutionState.Unlocked)

    def unlock_solution(self, key):
        if self.is_solution_locked():
            unscrambled = unscramble_solution(self.solution, self.width, self.height, key,
                                              ignore_chars=self.blacksquare())
            if not self.check_answers(unscrambled):
                return False

            # clear the scrambled bit and cksum
            self.solution = unscrambled
            self.scrambled_cksum = 0
            self.solution_state = SolutionState.Unlocked

        return True

    def lock_solution(self, key):
        if not self.is_solution_locked():
            # set the scrambled bit and cksum
            self.scrambled_cksum = scrambled_cksum(self.solution, self.width, self.height,
                                                   ignore_chars=self.blacksquare(), encoding=self.encoding)
            self.solution_state = SolutionState.Locked
            scrambled = scramble_solution(self.solution, self.width, self.height, key,
                                          ignore_chars=self.blacksquare())
            self.solution = scrambled

    def check_answers(self, fill):
        if self.is_solution_locked():
            scrambled = scrambled_cksum(fill, self.width, self.height,
                                        ignore_chars=self.blacksquare(), encoding=self.encoding)
            return scrambled == self.scrambled_cksum
        else:
            return fill == self.solution

    def header_cksum(self, cksum=0):
        return data_cksum(struct.pack(HEADER_CKSUM_FORMAT,
                          self.width, self.height, len(self.clues),
                          self.puzzletype, self.solution_state), cksum)

    def text_cksum(self, cksum=0):
        # for the checksum to work these fields must be added in order with
        # null termination, followed by all non-empty clues without null
        # termination, followed by notes (but only for version >= 1.3)
        if self.title:
            cksum = data_cksum(self.encode_zstring(self.title), cksum)
        if self.author:
            cksum = data_cksum(self.encode_zstring(self.author), cksum)
        if self.copyright:
            cksum = data_cksum(self.encode_zstring(self.copyright), cksum)

        for clue in self.clues:
            if clue:
                cksum = data_cksum(self.encode(clue), cksum)

        # notes included in global cksum starting v1.3 of format
        if self.version_tuple() >= (1, 3) and self.notes:
            cksum = data_cksum(self.encode_zstring(self.notes), cksum)

        return cksum

    def global_cksum(self):
        cksum = self.header_cksum()
        cksum = data_cksum(self.encode(self.solution), cksum)
        cksum = data_cksum(self.encode(self.fill), cksum)
        cksum = self.text_cksum(cksum)
        # extensions do not seem to be included in global cksum
        return cksum

    def magic_cksum(self):
        cksums = [
            self.header_cksum(),
            data_cksum(self.encode(self.solution)),
            data_cksum(self.encode(self.fill)),
            self.text_cksum()
        ]

        cksum_magic = 0
        for (i, cksum) in enumerate(reversed(cksums)):
            cksum_magic <<= 8
            cksum_magic |= (
                ord(MASKSTRING[len(cksums) - i - 1]) ^ (cksum & 0x00ff)
            )
            cksum_magic |= (
                (ord(MASKSTRING[len(cksums) - i - 1 + 4]) ^ (cksum >> 8)) << 32
            )

        return cksum_magic

class PuzzleBuffer:
    """PuzzleBuffer class
    wraps a data buffer ('' or []) and provides .puz-specific methods for
    reading and writing data
    """
    def __init__(self, data=None, encoding=ENCODING):
        self.data = data or []
        self.encoding = encoding
        self.pos = 0

    def can_read(self, n_bytes=1):
        return self.pos + n_bytes <= len(self.data)

    def length(self):
        return len(self.data)

    def read(self, n_bytes):
        start = self.pos
        self.pos += n_bytes
        return self.data[start:self.pos]

    def read_to_end(self):
        start = self.pos
        self.pos = self.length()
        return self.data[start:self.pos]

    def read_string(self):
        return self.read_until(b'\0')

    def read_until(self, c):
        start = self.pos
        self.seek_to(c, 1)  # read past
        return str(self.data[start:self.pos-1], self.encoding)

    def seek_to(self, s, offset=0):
        try:
            self.pos = self.data.index(s, self.pos) + offset
            return True
        except ValueError:
            # s not found, advance to end
            self.pos = self.length()
            return False

    def write(self, s):
        self.data.append(s)

    def write_string(self, s):
        s = s or ''
        self.data.append(s.encode(self.encoding, ENCODING_ERRORS) + b'\0')

    def pack(self, struct_format, *values):
        self.data.append(struct.pack(struct_format, *values))

    def can_unpack(self, struct_format):
        return self.can_read(struct.calcsize(struct_format))

    def unpack(self, struct_format):
        start = self.pos
        try:
            res = struct.unpack_from(struct_format, self.data, self.pos)
            self.pos += struct.calcsize(struct_format)
            return res
        except struct.error:
            message = 'could not unpack values at {} for format {}'.format(
                start, struct_format
            )
            raise PuzzleFormatError(message)

    def tobytes(self):
        return b''.join(self.data)

# clue numbering helper

class DefaultClueNumbering:
    def __init__(self, grid, clues, width, height):
        self.grid = grid
        self.clues = clues
        self.width = width
        self.height = height

        # compute across & down
        a = []
        d = []
        c = 0
        n = 1
        for i in range(0, len(grid)):
            if not is_blacksquare(grid[i]):
                lastc = c
                is_across = self.col(i) == 0 or is_blacksquare(grid[i - 1])
                if is_across and self.len_across(i) > 1:
                    a.append({
                        'num': n,
                        'clue': clues[c],
                        'clue_index': c,
                        'cell': i,
                        'len': self.len_across(i)
                    })
                    c += 1
                is_down = self.row(i) == 0 or is_blacksquare(grid[i - width])
                if is_down and self.len_down(i) > 1:
                    d.append({
                        'num': n,
                        'clue': clues[c],
                        'clue_index': c,
                        'cell': i,
                        'len': self.len_down(i)
                    })
                    c += 1
                if c > lastc:
                    n += 1

        self.across = a
        self.down = d

    def col(self, index):
        return index % self.width

    def row(self, index):
        return int(math.floor(index / self.width))

    def len_across(self, index):
        for c in range(0, self.width - self.col(index)):
            if is_blacksquare(self.grid[index + c]):
                return c
        return c + 1

    def len_down(self, index):
        for c in range(0, self.height - self.row(index)):
            if is_blacksquare(self.grid[index + c*self.width]):
                return c
        return c + 1

class Rebus:
    def __init__(self, puzzle):
        self.puzzle = puzzle
        self.temp = {}
        # parse rebus data
        rebus_data = self.puzzle.extensions.get(Extensions.Rebus, b'')
        self.table = parse_bytes(rebus_data)
        r_sol_data = self.puzzle.extensions.get(Extensions.RebusSolutions, b'')
        solutions_str = r_sol_data.decode(puzzle.encoding)
        fill_data = self.puzzle.extensions.get(Extensions.RebusFill, b'')
        fill_str = fill_data.decode(puzzle.encoding)
        self.solutions = dict(
            (int(item[0]), item[1])
            for item in parse_dict(solutions_str).items()
        )
        self.fill = dict(
            (int(item[0]), item[1])
            for item in parse_dict(fill_str).items()
        )

    def add_rebus(self, answer):
        if answer is None:
            self.table.append(0)
        else:
            if answer not in self.temp:
                self.temp[answer] = len(self.temp) + 1
            self.solutions[f"{self.temp[answer]:2d}"] = answer
            self.table.append(self.temp[answer] + 1)

    def has_rebus(self):
        return Extensions.Rebus in self.puzzle.extensions

    def is_rebus_square(self, index):
        return bool(self.table[index])

    def get_rebus_squares(self):
        return [i for i, b in enumerate(self.table) if b]

    def get_rebus_solution(self, index):
        if self.is_rebus_square(index):
            return self.solutions[self.table[index] - 1]
        return None

    def get_rebus_fill(self, index):
        if self.is_rebus_square(index):
            return self.fill[self.table[index] - 1]
        return None

    def set_rebus_fill(self, index, value):
        if self.is_rebus_square(index):
            self.fill[self.table[index] - 1] = value

    def save(self):
        if self.has_rebus():
            # commit changes back to puzzle.extensions
            self.puzzle.extensions[Extensions.Rebus] = pack_bytes(self.table)
            rebus_solutions = self.puzzle.encode(dict_to_string(self.solutions))
            self.puzzle.extensions[Extensions.RebusSolutions] = rebus_solutions
            # rebus_fill = self.puzzle.encode(dict_to_string(self.fill))
            # self.puzzle.extensions[Extensions.RebusFill] = rebus_fill

class Markup:
    def __init__(self, puzzle):
        self.puzzle = puzzle
        # parse markup data
        markup_data = self.puzzle.extensions.get(Extensions.Markup, b'')
        self.markup = parse_bytes(markup_data)

    def has_markup(self):
        return any(bool(b) for b in self.markup)

    def get_markup_squares(self):
        return [i for i, b in enumerate(self.markup) if b]

    def is_markup_square(self, index):
        return bool(self.markup[index])

    def save(self):
        if self.has_markup():
            self.puzzle.extensions[Extensions.Markup] = pack_bytes(self.markup)

# helper functions for cksums and scrambling
def data_cksum(data, cksum=0):
    for b in data:
        if isinstance(b, bytes):
            b = ord(b)
        # right-shift one with wrap-around
        lowbit = (cksum & 0x0001)
        cksum = (cksum >> 1)
        if lowbit:
            cksum = (cksum | 0x8000)

        # then add in the data and clear any carried bit past 16
        cksum = (cksum + b) & 0xffff

    return cksum

def replace_chars(s, chars, replacement=''):
    for ch in chars:
        s = s.replace(ch, replacement)
    return s

def scramble_solution(solution, width, height, key, ignore_chars=BLACKSQUARE):
    sq = square(solution, width, height)
    data = restore(sq, scramble_string(replace_chars(sq, ignore_chars), key))
    return square(data, height, width)

def scramble_string(s, key):
    """
    s is the puzzle's solution in column-major order, omitting black squares:
    i.e. if the puzzle is:
        C A T
        # # A
        # # R
    solution is CATAR
    Key is a 4-digit number in the range 1000 <= key <= 9999
    """
    key = key_digits(key)
    for k in key:          # foreach digit in the key
        s = shift(s, key)  # for each char by each digit in the key in sequence
        s = s[k:] + s[:k]  # cut the sequence around the key digit
        s = shuffle(s)     # do a 1:1 shuffle of the 'deck'

    return s

def unscramble_solution(scrambled, width, height, key, ignore_chars=BLACKSQUARE):
    # width and height are reversed here
    sq = square(scrambled, width, height)
    data = restore(sq, unscramble_string(replace_chars(sq, ignore_chars), key))
    return square(data, height, width)

def unscramble_string(s, key):
    key = key_digits(key)
    l = len(s)  # noqa: E741
    for k in key[::-1]:
        s = unshuffle(s)
        s = s[l-k:] + s[:l-k]
        s = unshift(s, key)

    return s

def scrambled_cksum(scrambled, width, height, ignore_chars=BLACKSQUARE, encoding=ENCODING):
    data = replace_chars(square(scrambled, width, height), ignore_chars)
    return data_cksum(data.encode(encoding, ENCODING_ERRORS))

def key_digits(key):
    return [int(c) for c in str(key).zfill(4)]

def square(data, w, h):
    aa = [data[i:i+w] for i in range(0, len(data), w)]
    return ''.join(
        [''.join([aa[r][c] for r in range(0, h)]) for c in range(0, w)]
    )

def shift(s, key):
    atoz = string.ascii_uppercase
    return ''.join(
        atoz[(atoz.index(c) + key[i % len(key)]) % len(atoz)]
        for i, c in enumerate(s)
    )

def unshift(s, key):
    return shift(s, [-k for k in key])

def shuffle(s):
    mid = int(math.floor(len(s) / 2))
    items = functools.reduce(operator.add, zip(s[mid:], s[:mid]))
    return ''.join(items) + (s[-1] if len(s) % 2 else '')

def unshuffle(s):
    return s[1::2] + s[::2]

def restore(s, t):
    """
    s is the source string, it can contain '.'
    t is the target, it's smaller than s by the number of '.'s in s
    Each char in s is replaced by the corresponding
    char in t, jumping over '.'s in s.
    >>> restore('ABC.DEF', 'XYZABC')
    'XYZ.ABC'
    """
    t = (c for c in t)
    return ''.join(next(t) if not is_blacksquare(c) else c for c in s)

def is_blacksquare(c):
    if isinstance(c, int):
        c = chr(c)
    return c in [BLACKSQUARE, BLACKSQUARE2]

#
# functions for parsing / serializing primitives
#

def parse_bytes(s):
    return list(struct.unpack('B' * len(s), s))

def pack_bytes(a):
    return struct.pack('B' * len(a), *a)

# dict string format is k1:v1;k2:v2;...;kn:vn;
# (for whatever reason there's a trailing ';')
def parse_dict(s):
    return dict(p.split(':') for p in s.split(';') if ':' in p)

def dict_to_string(d):
    return ';'.join(':'.join(map(str, [k, v])) for k, v in d.items()) + ';'


text:
# How to install and use

![Releases link](images/github.50.png)

First off, click on the releases link

![Release zip file](images/releases.50.png)

Then click on the release zip file to download it.

![Several files](images/explorer.50.png)

If you open the zip file, you'll see it contains several files. You want to extract all of the files to a directory on your local computer.

Once done, double click on `nytxw_puz.exe`

![Console window asking three questions](images/console.50.png)

That will open a console window asking three questions.  Pick the browser you're logged into NY Times with (I used Chrome here), and copy the URL from the puzzle you want to convert, by copy-n-pasting it from the website:

![Website URL](images/nytimes.50.png)

And type in the output filename.

Once it's done, the file will be created, and you can open it in your favorite application:

![Tada](images/final_results.50.png)

repo_language: Python


text:
#!/usr/bin/env python3

from email.quoprimime import unquote
import browser_cookie3
import requests
import requests.utils
import os
import decompress
import re
import json
import puz
import sys
import html
import datetime
import time
import base64
import version

CACHE_DATA = False
# Debug code to log all URL requests and their responses, set to None
# to disable logging, any string enables (and is replaced with a log filename)
LOG_CALLS = None

# These unicode characters are just used to draw the crossword grid to stdout
BLOCK_LEFT = "\u2590"
BLOCK_MID = "\u2588"
BLOCK_RIGHT = "\u258c"
TITLE_LINE = "\u2501"

# Different possibilities for each cell in the NYT's JSON data structure
NYT_TYPE_BLOCK = 0     # Black cell, no clues or answer
NYT_TYPE_NORMAL = 1    # Normal cell, could be a rebus
NYT_TYPE_CIRCLED = 2   # Cell with a circle around it as for letters part of a theme
NYT_TYPE_GRAY = 3      # A cell filled in as gray
NYT_TYPE_INVISIBLE = 4 # An "invisible" cell, generally something outside the main grid

LATIN1_SUBS = {
    # For converting clues etc. into Latin-1 (ISO-8859-1) format;
    # value None means let the encoder insert a Latin-1 equivalent
    u"\n": u" ",
    u"\r": u" ",
    u'“': u'"',
    u'”': u'"',
    u'‘': u"'",
    u'’': u"'",
    u'–': u'-',
    u'—': u'--',
    u'…': u'...',
    u'№': u'No.',
    u'π': u'pi',
    u'€': u'EUR',
    u'•': u'*',
    u'†': u'[dagger]',
    u'‡': u'[double dagger]',
    u'™': u'[TM]',
    u'‹': u'<',
    u'›': u'>',
    u'←': u'<--',
    u'■': None,
    u'☐': None,
    u'→': u'-->',
    u'♣': "Clubs",
    u'√': None,
    u'♠': "Spades",
    u'✓': None,
    u'♭': None,
    u'♂': None,
    u'★': u'*',
    u'θ': u'theta',
    u'β': u'beta',
    u'Č': None,
    u'𝚫': u'Delta',
    u'❤': None,
    u'✔': None,
    u'⚓': None,
    u'♦': "Diamonds",
    u'♥': "Hearts",
    u'☹': None,
    u'☮': None,
    u'☘': None,
    u'◯': None,
    u'▢': None,
    u'∑': None,
    u'∃': None,
    u'↓': None,
    u'⁎': u'*',
    u'η': u'eta',
    u'α': u'alpha',
    u'Ω': u'Omega',
    u'ō': None,
}
# Some rules to remove HTML like things with text versions for the .puz files
HTML_TO_TEXT_RULES = [
    ("(.*?)", "_\\1_"),              # "Italic" -> "_Italic_"
    ("(.*?)", "_\\1_"),            # "Italic" -> "_Italic_"
    ("(.*?)", "\\1"),            # "KNO3" -> "KNO3"
    ("([0-9 ]+)", "^\\1"),       # "E=MC2" -> "E=MC^2"
    ("(.*?)", "\\1"),            # "103rd" -> "103rd" (Note, after the numeric 'sup')
    ("
", " / "),                   # "A
B
C" -> "A / B / C"
    ("<s>(.*?)</s>", "[*cross out* \\1]"),  # "<s>Crossed Out</s>" -> "[*cross out* Crossed out]"
    ("<[^>]+>", ""),                        # Remove other things that look like HTML, but leave bare "<" alone.
    (" ", " "),                        # Replace HTML's non-breaking spaces into normal spaces
]

# Work around https://github.com/borisbabic/browser_cookie3/issues/104
def chrome_bugfix(**kargs):
    args = {
        'linux_cookies':[
                '~/.config/google-chrome/Default/Cookies',
                '~/.config/google-chrome-beta/Default/Cookies'
            ],
        'windows_cookies':[
                {'env':'APPDATA', 'path':'..\\Local\\Google\\Chrome\\User Data\\Default\\Cookies'},
                {'env':'APPDATA', 'path':'..\\Local\\Google\\Chrome\\User Data\\Default\\Network\\Cookies'},
                {'env':'LOCALAPPDATA', 'path':'Google\\Chrome\\User Data\\Default\\Cookies'},
                {'env':'LOCALAPPDATA', 'path':'Google\\Chrome\\User Data\\Default\\Network\\Cookies'},
                {'env':'APPDATA', 'path':'Google\\Chrome\\User Data\\Default\\Cookies'},
                {'env':'APPDATA', 'path':'Google\\Chrome\\User Data\\Default\\Network\\Cookies'}
            ],
        'osx_cookies': ['~/Library/Application Support/Google/Chrome/Default/Cookies'],
        'windows_keys': [
                {'env':'APPDATA', 'path':'..\\Local\\Google\\Chrome\\User Data\\Local State'},
                {'env':'LOCALAPPDATA', 'path':'Google\\Chrome\\User Data\\Local State'},
                {'env':'APPDATA', 'path':'Google\\Chrome\\User Data\\Local State'}
            ],
        'os_crypt_name':'chrome',
        'osx_key_service' : 'Chrome Safe Storage',
        'osx_key_user' : 'Chrome'
    }
    for key, value in kargs.items():
        args[key] = value

    cookies = browser_cookie3.ChromiumBased(browser='Chrome', **args)
    return cookies.load()

# Work around https://github.com/borisbabic/browser_cookie3/issues/104
def chromium_bugfix(**kargs):
    args = {
        'linux_cookies':['~/.config/chromium/Default/Cookies'],
        'windows_cookies':[
                {'env':'APPDATA', 'path':'..\\Local\\Chromium\\User Data\\Default\\Cookies'},
                {'env':'APPDATA', 'path':'..\\Local\\Chromium\\User Data\\Default\\Network\\Cookies'},
                {'env':'LOCALAPPDATA', 'path':'Chromium\\User Data\\Default\\Cookies'},
                {'env':'LOCALAPPDATA', 'path':'Chromium\\User Data\\Default\\Network\\Cookies'},
                {'env':'APPDATA', 'path':'Chromium\\User Data\\Default\\Cookies'},
                {'env':'APPDATA', 'path':'Chromium\\User Data\\Default\\Network\\Cookies'}
        ],
        'osx_cookies': ['~/Library/Application Support/Chromium/Default/Cookies'],
        'windows_keys': [
                {'env':'APPDATA', 'path':'..\\Local\\Chromium\\User Data\\Local State'},
                {'env':'LOCALAPPDATA', 'path':'Chromium\\User Data\\Local State'},
                {'env':'APPDATA', 'path':'Chromium\\User Data\\Local State'}
        ],
        'os_crypt_name':'chromium',
        'osx_key_service' : 'Chromium Safe Storage',
        'osx_key_user' : 'Chromium'
    }
    for key, value in kargs.items():
        args[key] = value

    cookies = browser_cookie3.ChromiumBased(browser='Chromium', **args)
    return cookies.load()

def get_browsers():
    return {
        # Work around https://github.com/borisbabic/browser_cookie3/issues/104
        # "Chrome": browser_cookie3.chrome,
        # "Chromium": browser_cookie3.chromium,
        "Chrome": chrome_bugfix,
        "Chromium": chromium_bugfix,

        "Opera": browser_cookie3.opera,
        "Microsoft Edge": browser_cookie3.edge,
        "Firefox": browser_cookie3.firefox,
    }

def pick_browser():
    # Ask the user for which browser they want to use
    if os.name == "nt":
        homedir = os.environ.get("APPDATA", os.path.expanduser("~"))
    else:
        homedir = format(os.path.expanduser("~"))

    options = get_browsers()
    settings_file = os.path.join(homedir, 'nytxw_puz.json')
    settings = {}
    if os.path.isfile(settings_file):
        with open(settings_file) as f:
            settings = json.load(f)

    default = settings.get('default')
    while True:
        default_number = None
        keys = list(sorted(options))
        for i, desc in enumerate(keys):
            print(f" {i+1}) {desc}")
            if default is not None and desc == default:
                default_number = str(i + 1)

        selection = input(f"Please select a browser that's logged into NYTimes.com{' [' + default_number + ']' if default_number else ''}: ")
        try:
            if len(selection) == 0 and default_number:
                selection = default_number
            selection = int(selection)
            if selection >= 1 and selection <= len(options):
                selection = keys[selection - 1]
                break
        except:
            pass

    settings['default'] = selection
    if default != selection:
        with open(settings_file, 'wt') as f:
            json.dump(settings, f)

    return selection

# Internal helper to load a URL, optionally log the data, to make
# debugging remotely a tiny bit easier
def get_url(cookies, url):
    cookies = requests.utils.dict_from_cookiejar(cookies)
    cookies = requests.utils.cookiejar_from_dict(cookies)
    if LOG_CALLS is not None:
        with open(LOG_CALLS, "a", newline="", encoding="utf-8") as f:
            f.write("URL " + url + "\n")
            f.write("COOKIES " + json.dumps(cookies, default=str) + "\n")
            try:
                f.write("COOKIES_RAW " + json.dumps(requests.utils.dict_from_cookiejar(cookies)) + "\n")
            except:
                pass
    resp = requests.get(url, cookies=cookies).content
    if LOG_CALLS is not None:
        with open(LOG_CALLS, "a", newline="", encoding="utf-8") as f:
            f.write("RESPONSE " + base64.b64encode(resp).decode("utf-8") + "\n")
    resp = resp.decode("utf-8")
    return resp

def load_cookies(browser):
    # Pull out the nytimes cookies from the user's browser
    cookies = get_browsers()[browser](domain_name='nytimes.com')
    return cookies

def get_puzzle_from_id(cookies, puzzle_id):
    # Get the puzzle itself
    puzzle_url = f"https://nyt-games-prd.appspot.com/svc/crosswords/v6/puzzle/{puzzle_id}.json"
    new_format = get_url(cookies, puzzle_url)
    new_format = json.loads(new_format)

    # The response is formatted somewhat differently than it used to be, so create a format
    # that looks like it used to
    resp = new_format["body"][0]
    resp["meta"] = {}
    # TODO: Notes might be stored elsewhere, need to verify
    for cur in ["publicationDate", "title", "editor", "copyright", "constructors", "notes"]:
        if cur in new_format:
            resp["meta"][cur] = new_format[cur]
    resp["dimensions"]["columnCount"] = resp["dimensions"]["width"]
    resp["dimensions"]["rowCount"] = resp["dimensions"]["height"]

    resp["gamePageData"] = resp

    return resp

def get_puzzle(url, browser):
    cache = {}
    if CACHE_DATA:
        # Simple cache, useful for debugging, grows to stupid
        # size over time, so it's off generally
        if os.path.isfile(".cached.json"):
            with open(".cached.json", "r", encoding="utf-8") as f:
                cache = json.load(f)

    if url not in cache:
        print(f"Loading {url}...")

        cookies = load_cookies(browser)
        for _ in range(4):
            # Load the webpage, its inline javascript includes the puzzle data
            resp = get_url(cookies, url)

            # NY Times is moving to a new system for puzzles, handle both, since
            # it doesn't seem to have migrated 100% of the accounts out there

            # Option #1, see if this is the old style encoded javascript blob:
            # Look for the javascript, it's easist here to just use a regex
            m = re.search("(pluribus|window.gameData) *= *['\"](?P<data>.*?)['\"]", resp)
            if m is not None:
                # Pull out the data element
                resp = m.group("data")
                if "%" in resp:
                    # Which is url-encoded
                    resp = decompress.decode(resp)
                    # And LZString compressed
                    resp = decompress.decompress(resp)
                else:
                    # New format, this is now base64 encoded
                    resp = base64.b64decode(resp).decode("utf-8")
                    # And _then_ url-encoded
                    resp = decompress.decode(resp)
                # And a JSON blob
                resp = json.loads(resp)
                # All done, we can stop retries
                break

            # Option #2, try the new version with a gaming REST endpoint:
            # Try to find the puzzle description:
            m = re.search("window\\.gameData *= *(?P<json>{.*?})", resp)
            if m is not None:
                # Pull out the puzzle key
                key = m.group("json")
                key = json.loads(key)
                key = key['filename']

                # Request the puzzle meta-data
                api = f"https://nyt-games-prd.appspot.com/svc/crosswords/v6/puzzle/{key}.json"
                metadata = get_url(cookies, api)
                metadata = json.loads(metadata)

                resp = get_puzzle_from_id(cookies, metadata['id'])

                # All done
                break

            # Something didn't look right, try again
            time.sleep(1)

        cache[url] = resp
        if CACHE_DATA:
            with open(".cached.json", "w", newline="", encoding="utf-8") as f:
                json.dump(cache, f)

    return cache[url]

def print_puzzle(p):
    # Dump out the puzzle, just a helper mostly to debug things
    p = p['gamePageData']
    width, height = p["dimensions"]["columnCount"], p["dimensions"]["rowCount"]
    for y in range(height):
        row = " "
        extra = ""
        shown = ""
        for x in range(width):
            cell = y * width + x
            cell = p["cells"][cell]
            if 'moreAnswers' in cell:
                # This is an oddball answer, note all the possibilities
                row += "- "
                temp = []
                if 'answer' in cell:
                    temp += [cell['answer']]
                temp += cell['moreAnswers']['valid']
                temp = f" ({', '.join(temp)})"
                if temp != shown:
                    shown = temp
                    extra += temp
            elif 'answer' in cell:
                # Normal answer, but if it's a rebus answer, show the first character
                # and the rebus answer to the side
                if len(cell['answer']) > 1:
                    extra += " " + cell['answer']
                    row += cell['answer'][0].lower() + " "
                else:
                    row += cell['answer'] + " "
            else:
                # Non-clue cell, just mark it
                row += "# "

        # Turn the "#" into block characters
        for x in range(len(row), 0, -1):
            row = row.replace(" " + "# " * x, BLOCK_LEFT + BLOCK_MID.join([BLOCK_MID] * x) + BLOCK_RIGHT)

        # And output the results
        print(" " + row + extra)

def latin1ify(s):
    source_string = s

    # Make a Unicode string compliant with the Latin-1 (ISO-8859-1) character
    # set; the Across Lite v1.3 format only supports Latin-1 encoding

    # Use table to convert the most common Unicode glyphs
    for search, replace in LATIN1_SUBS.items():
        if replace is not None:
            s = s.replace(search, replace)

    # Convert anything remaining using replacements like '\N{WINKING FACE}'
    s = s.encode('ISO-8859-1', 'namereplace').decode('ISO-8859-1')

    # Replace HTML like things into plain text
    for pattern, repl in HTML_TO_TEXT_RULES:
        s = re.sub(pattern, repl, s)

    s = s.strip()

    # Warn on anything left over
    for x in s:
        if not ' ' <= x <= '~':
            print(f"Warning: {json.dumps(x)} will likely cause problems in {s} from {source_string}")

    return s

def gridchar(c):
    if 'answer' in c:
        # The usual case: just one letter
        return latin1ify(c['answer'][0])
    if 'moreAnswers' in c:
        # 2022-05-01 'Blank Expressions' includes grid answers without
        # an actual 'answer', only an array of 'moreAnswers'
        more = c.get('moreAnswers', [])
        if isinstance(more, dict):
            more = more['valid']

        for a in more:
            if len(a) == 1:
                # First single-character one
                return latin1ify(a)
        # No single-character answers
        return 'X'

    # Black square
    return '.'

def gridrebus(c):
    if 'answer' in c:
        if len(c['answer']) > 1:
            # This cell has a rebus answer, but first, see if we can find a
            # answer that's already easy to use
            more = c.get('moreAnswers', [])
            if isinstance(more, dict):
                more = more['valid']
            answers = [c['answer']] + more
            for possible in answers:
                if possible == latin1ify(possible) and len(possible) > 1:
                    # This is a possibility that works well
                    return possible
            # Nothing useful, just use the first clue
            return answers[0]
    return None

def data_to_puz(puzzle):
    p = puz.Puzzle()
    data = puzzle['gamePageData']

    # Basic header
    p.title = 'New York Times Crossword'
    if 'publicationDate' in data['meta']:
        year, month, day = data['meta']['publicationDate'].split('-')
        d = datetime.date(int(year), int(month), int(day))
        months = ['', 'January', 'February', 'March', 'April', 'May', 'June',
                  'July', 'August', 'September', 'October', 'November', 'December']
        dow = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday',
               'Saturday', 'Sunday']
        p.title = ('NY Times, ' + dow[d.weekday()] + ', ' + months[d.month] +
                   ' ' + str(d.day) + ', ' + str(d.year))
        if 'title' in data['meta']:
            p.title += ' ' + latin1ify(data['meta']['title'].upper())
    elif 'title' in data['meta']:
        p.title = latin1ify(data['meta']['title'].upper())
    p.author = ', '.join(latin1ify(c) for c in data['meta']['constructors'])
    if 'editor' in data['meta']:
        p.author += ' / ' + latin1ify(data['meta']['editor'])
    if 'copyright' in data['meta']:
        p.copyright = '© ' + data['meta']['copyright'] + ', The New York Times'

    # Pull out the size of the puzzle
    p.height = data["dimensions"]["rowCount"]
    p.width = data["dimensions"]["columnCount"]

    # Fill out the main grid
    p.solution = ''.join(gridchar(x) for x in data['cells'])
    p.fill = ''.join('-' if 'answer' in x else '.' for x in data['cells'])

    # And the clues, they're HTML text here, so decode them, Across Lite expects them in
    # crossword order, not the NYT clue order, order them correctly
    seen = set()
    clues = []
    for cell in data['cells']:
        for clue in cell.get('clues', []):
            if clue not in seen:
                seen.add(clue)
                temp = data['clues'][clue]['text']
                if isinstance(temp, list):
                    temp = temp[0]
                if isinstance(temp, dict):
                    temp = temp.get("plain", "")
                clues.append(latin1ify(html.unescape(temp)))
    p.clues = clues

    # See if any of the answers is multi-character (rebus)
    if max([len(x['answer']) for x in data['cells'] if 'answer' in x]) > 1:
        # We have at least one rebus answer, so setup the rebus data fields
        rebus = p.create_empty_rebus()

        # And find all the rebus answers and add them to the data
        for cell in data['cells']:
            rebus.add_rebus(gridrebus(cell))

    # See if any grid squares are marked up with circles
    if any(x['type'] in (NYT_TYPE_CIRCLED, NYT_TYPE_GRAY) for x in data['cells'] if 'type' in x):
        markup = p.markup()
        markup.markup = [0] * (p.width * p.height)

        for i, cell in enumerate(data['cells']):
            if 'type' in cell and cell['type'] in (NYT_TYPE_CIRCLED, NYT_TYPE_GRAY):
                markup.markup[cell.get('index', i)] = puz.GridMarkup.Circled

    # Check for any notes in puzzle (e.g., Sep 11, 2008)
    if data['meta'].get('notes', None) is not None:
        p.notes = '\n\n'.join(latin1ify(x['text']) for x in data['meta']['notes']
                              if 'text' in x)

    # All done
    return p

def version_warn():
    ver = version.get_ver_from_github()
    if ver is not None:
        if ver > version.VERSION:
            print(f"Info: Version {ver} is available, consider upgrading if any issues occur")
            print("")

def main():
    if len(sys.argv) == 4:
        browser, url, output_fn = sys.argv[1:4]
        if browser not in get_browsers():
            print(f"ERROR: Unknown browser '{browser}', please select one of:")
            for key in get_browsers():
                print(f"  {key}")
            exit(1)
    else:
        version_warn()
        browser = pick_browser()
        url = input("Enter the NY Times crossword URL: ")
        if len(url) == 0:
            print("No URL specified")
            exit(1)
        output_fn = input("Enter the output filename: ")
        if len(url) == 0:
            print("No output file specified")
            exit(1)

    global LOG_CALLS
    if LOG_CALLS is not None:
        LOG_CALLS = output_fn + ".log"
        with open(LOG_CALLS, "a", newline="", encoding="utf-8") as f:
            f.write("LOG " + datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S") + "\n")

    try:
        # url = "https://www.nytimes.com/crosswords/game/daily/2021/06/03"
        # output_fn = "test.puz"

        # Get the puzzle from NYT, the first time this is called
        # the cookie will be archived
        puzzle = get_puzzle(url, browser)

        # Useful for debugging, hidden by default since
        # showing the solution kinda defeats the point
        # print_puzzle(puzzle)

        # And turn the puzzle data from NYT into a puz data structure
        output = data_to_puz(puzzle)
        output.save(output_fn)
        print(f"Created {output_fn}")
    except:
        print("ERROR! " * 10)
        import traceback
        traceback.print_exc()
        print("ERROR! " * 10)
        print("Settings: ", [browser, url, output_fn, version.VERSION])
        print("")
        print("Please report issues to https://www.reddit.com/user/nobody514/")
        print("")
        input("Press enter to continue...")

if __name__ == "__main__":
    main()


text:
altgraph==0.17.3
browser-cookie3==0.17.1
cachetools==5.3.0
certifi==2022.12.7
cffi==1.15.1
chardet==5.1.0
charset-normalizer==3.1.0
cryptography==40.0.2
fresh==0.0.1
future==0.18.3
idna==3.4
importlib-metadata==6.4.1
jaraco.classes==3.2.3
jeepney==0.8.0
keyring==23.13.1
lz4==4.3.2
more-itertools==9.1.0
pbkdf2==1.3
pefile==2023.2.7
py2exe==0.13.0.0; sys_platform == 'win32'
pyaes==1.6.1
pycparser==2.21
pycryptodome==3.17
pycryptodomex==3.17
pyinstaller==5.10.1
pyinstaller-hooks-contrib==2023.2
pywin32-ctypes==0.2.0
requests==2.28.2
SecretStorage==3.3.3
urllib3==1.26.15
zipp==3.15.0


text:
#!/usr/bin/env python3

from distutils.core import setup
from setuptools import find_packages
import py2exe

py2exe.freeze(
    console=[{
        'script': 'nyt.py',
        'dest_base': 'nytxw_puz',
        'product_name': 'nytxw_puz',
    }],
    options={
        'py2exe': {
            'bundle_files': 1,
            'dist_dir': 'dist',
        }
    },
    zipfile=None,
)


text:
#
# Build: podman build -t nytxw .
#
# Run: podman run --rm -v $HOME/.mozilla:/root/.mozilla:Z,ro \
#                 -v $HOME/path/to/nyt:/xw:Z \
#                  nytxw Firefox \
#                        $(date +'https://www.nytimes.com/crosswords/game/daily/%Y/%m/%d') \
#                        /xw/$(date --iso-8601=date).puz
#
FROM docker.io/library/alpine:latest
WORKDIR /nytxw_puz
ADD *.py *.txt /nytxw_puz
RUN apk update && \
    apk add python3 py3-pip py3-wheel python3-dev gcc musl-dev libffi-dev zlib-dev rust cargo openssl-dev && \
    python3 -m pip install -r requirements.txt && \
    apk del py3-wheel python3-dev gcc musl-dev libffi-dev zlib-dev rust cargo openssl-dev && \
    rm -rf /root/.cargo
ENTRYPOINT ["/nytxw_puz/nyt.py"]


text:
# NYT Crossword to Puz

![Logo](other/logo.png)

A windows program to convert NY Times crosswords from the web to Across Lite compatible files.

To run this, first download and decompress the [release](https://github.com/Q726kbXuN/nytxw_puz/releases/latest/download/nytxw_puz.zip).  Then, run the executable, and answer the three questions:

```
 1) Chrome
 2) Chromium
 3) Firefox
 4) Microsoft Edge
 5) Opera
Please select a browser that's logged into NYTimes.com: 1
Enter the NY Times crossword URL: https://www.nytimes.com/crosswords/game/daily/1994/02/14
Enter the output filename: example.puz
Loading https://www.nytimes.com/crosswords/game/daily/1994/02/14...
Created example.puz
```

Note: Running this script requires you're logged into nytimes.com in your browser.

Alternatively, you can pass in the browser, URL, and filename from the command line.

To run the Python script directly, clone this repo, then setup the venv to download packages:
```
python -m venv .venv
.venv\Scripts\activate.bat
```

And install the packages:
```
python -m pip install -r requirements.txt
```

Then, to use:
```
nyt.py Chrome https://www.nytimes.com/crosswords/game/daily/2021/08/03 2021-08-03.puz
```

# Mini FAQ

* What? Why?

The New York Times announced that on August 10th, 2021, [they will no longer provide](https://www.nytimes.com/2021/08/02/crosswords/nyt-games-no-longer-available-on-across-lite-as-of-aug-9.html) Across Lite .puz files for download.  This tool was made in response to that decision, allowing users of that tool to continue to solve crossword puzzles in their favorite program.

* Why not just use the NY Times app?

For me personally, I want something that works completely offline, and doesn't require I stare at my phone even more.  I'm sure other people have other reasons, so I decided to put my little tool online for others to use.

* Help?  How do I run this?

Try following [these directions](howto.md).

* I downloaded this a while ago and it stopped working!

Please update to at least version 1.07.  There was a change with how the puzzles are published that broke older versions of this program.

* Anything else?

Feel free to contact me on [reddit](https://www.reddit.com/user/nobody514/).


text:
#!/usr/bin/env python3

import os
import sys
import inspect
currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
parentdir = os.path.dirname(currentdir)
sys.path.insert(0, parentdir)
import nyt

puzzle_data = """
. . . x x x x a x x x . . . x x x
n y t i m e s c r o s s w o r d s
x x x x x x x r . . x x e x x x .
x x x x x x x o . x x x b x x x .
x x . . . . x s . x x x p x x x .
x x . x x x x s x x x x a x x x x
x x . x x x x l x x x x g x . x x
x x x x x x x i x x x x e x . x x
. c o n v e r t . x x . . . . x x
. x x x x x x e . x x x x x x x x
. x x x x x x . . x x x x x x x x
x x x . . . x x x x x x x x . . .
"""

def add_clue(x, y, puzzle, x_dir, y_dir):
    puzzle["temp"][(x, y, x_dir, y_dir)] = len(puzzle['clues'])
    clue = ""
    while x < len(puzzle_data[0]) and y < len(puzzle_data) and puzzle_data[y][x] != ".":
        clue += puzzle_data[y][x]
        x += x_dir
        y += y_dir
    if "x" in clue:
        clue = "-"
    puzzle['clues'].append({"text": clue.upper()})

def main():
    global puzzle_data

    puzzle = {
        "dimensions": {
            "rowCount": 0,
            "columnCount": 0,
        },
        "cells": [],
        "clues": [
        ],
        "meta": {
            "constructors": [],
        },
        "temp": {},
    }
    output_fn = "logo_puzzle.puz"

    puzzle_data = [x.strip("\r\n").split(" ") for x in puzzle_data.split("\n") if len(x.strip("\r\n"))]

    for y in range(len(puzzle_data)):
        for x in range(len(puzzle_data[0])):
            if puzzle_data[y][x] != ".":
                if x < len(puzzle_data[0]) - 1:
                    if x == 0:
                        if puzzle_data[y][x + 1] != ".":
                            add_clue(x, y, puzzle, 1, 0)
                    else:
                        if puzzle_data[y][x-1] == "." and puzzle_data[y][x+1] != ".":
                            add_clue(x, y, puzzle, 1, 0)
                if y < len(puzzle_data) - 1:
                    if y == 0:
                        if puzzle_data[y + 1][x] != ".":
                            add_clue(x, y, puzzle, 0, 1)
                    else:
                        if puzzle_data[y-1][x] == "." and puzzle_data[y+1][x] != ".":
                            add_clue(x, y, puzzle, 0, 1)

    puzzle['dimensions']['columnCount'] = len(puzzle_data[0])
    puzzle['dimensions']['rowCount'] = len(puzzle_data)

    for y, row in enumerate(puzzle_data):
        for x, cell in enumerate(row):
            if cell == ".":
                puzzle['cells'].append({'clues': []})
            else:
                puzzle['cells'].append({'answer': cell.upper(), 'clues': []})
            if (x, y, 1, 0) in puzzle['temp']:
                puzzle['cells'][-1]['clues'].append(puzzle['temp'][(x, y, 1, 0)])
            if (x, y, 0, 1) in puzzle['temp']:
                puzzle['cells'][-1]['clues'].append(puzzle['temp'][(x, y, 0, 1)])

    puzzle['gamePageData'] = puzzle

    nyt.print_puzzle(puzzle)
    output = nyt.data_to_puz(puzzle)
    output.save(output_fn)
    print(f"Created {output_fn}")

if __name__ == "__main__":
    main()


text:
# -*- coding: utf-8 -*-

import base64
import configparser
import contextlib
import glob
import http.cookiejar
import json
import os
import struct
import subprocess
import sys
import tempfile
from io import BytesIO
from typing import Union

try:
    # should use pysqlite2 to read the cookies.sqlite on Windows
    # otherwise will raise the "sqlite3.DatabaseError: file is encrypted or is not a database" exception
    from pysqlite2 import dbapi2 as sqlite3
except ImportError:
    import sqlite3

if sys.platform.startswith('linux') or 'bsd' in sys.platform.lower():
    try:
        import dbus
        USE_DBUS_LINUX = True
    except ImportError:
        import jeepney
        from jeepney.io.blocking import open_dbus_connection
        USE_DBUS_LINUX = False

# external dependencies
import lz4.block

from Cryptodome.Cipher import AES
from Cryptodome.Protocol.KDF import PBKDF2
from Cryptodome.Util.Padding import unpad

__doc__ = 'Load browser cookies into a cookiejar'

CHROMIUM_DEFAULT_PASSWORD = b'peanuts'

class BrowserCookieError(Exception):
    pass

def _create_local_copy(cookie_file):
    """Make a local copy of the sqlite cookie database and return the new filename.
    This is necessary in case this database is still being written to while the user browses
    to avoid sqlite locking errors.
    """
    # check if cookie file exists
    if os.path.exists(cookie_file):
        # copy to random name in tmp folder
        tmp_cookie_file = tempfile.NamedTemporaryFile(suffix='.sqlite').name
        with open(tmp_cookie_file, "wb") as f1, open(cookie_file, "rb") as f2:
            f1.write(f2.read())
        return tmp_cookie_file
    else:
        raise BrowserCookieError('Can not find cookie file at: ' + cookie_file)

def _windows_group_policy_path():
    # we know that we're running under windows at this point so it's safe to do these imports
    from winreg import (HKEY_LOCAL_MACHINE, REG_EXPAND_SZ, REG_SZ,
                        ConnectRegistry, OpenKeyEx, QueryValueEx)
    try:
        root = ConnectRegistry(None, HKEY_LOCAL_MACHINE)
        policy_key = OpenKeyEx(root, r"SOFTWARE\Policies\Google\Chrome")
        user_data_dir, type_ = QueryValueEx(policy_key, "UserDataDir")
        if type_ == REG_EXPAND_SZ:
            user_data_dir = os.path.expandvars(user_data_dir)
        elif type_ != REG_SZ:
            return None
    except OSError:
        return None
    return os.path.join(user_data_dir, "Default", "Cookies")

# Code adapted slightly from https://github.com/Arnie97/chrome-cookies
def _crypt_unprotect_data(
        cipher_text=b'', entropy=b'', reserved=None, prompt_struct=None, is_key=False
):
    # we know that we're running under windows at this point so it's safe to try these imports
    import ctypes
    import ctypes.wintypes

    class DataBlob(ctypes.Structure):
        _fields_ = [
            ('cbData', ctypes.wintypes.DWORD),
            ('pbData', ctypes.POINTER(ctypes.c_char))
        ]

    blob_in, blob_entropy, blob_out = map(
        lambda x: DataBlob(len(x), ctypes.create_string_buffer(x)),
        [cipher_text, entropy, b'']
    )
    desc = ctypes.c_wchar_p()

    CRYPTPROTECT_UI_FORBIDDEN = 0x01

    if not ctypes.windll.crypt32.CryptUnprotectData(
            ctypes.byref(blob_in), ctypes.byref(
                desc), ctypes.byref(blob_entropy),
            reserved, prompt_struct, CRYPTPROTECT_UI_FORBIDDEN, ctypes.byref(
                blob_out)
    ):
        raise RuntimeError('Failed to decrypt the cipher text with DPAPI')

    description = desc.value
    buffer_out = ctypes.create_string_buffer(int(blob_out.cbData))
    ctypes.memmove(buffer_out, blob_out.pbData, blob_out.cbData)
    map(ctypes.windll.kernel32.LocalFree, [desc, blob_out.pbData])
    if is_key:
        return description, buffer_out.raw
    else:
        return description, buffer_out.value

def _get_osx_keychain_password(osx_key_service, osx_key_user):
    """Retrieve password used to encrypt cookies from OSX Keychain"""

    cmd = ['/usr/bin/security', '-q', 'find-generic-password', '-w', '-a', osx_key_user, '-s', osx_key_service]
    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    out, err = proc.communicate()
    if proc.returncode != 0:
        return CHROMIUM_DEFAULT_PASSWORD     # default password, probably won't work
    return out.strip()

def _expand_win_path(path:Union[dict,str]):
    if not isinstance(path,dict):
        path = {'path': path}
    return os.path.join(os.getenv(path['env'], ''), path['path'])

def _expand_paths_impl(paths:list, os_name:str):
    """Expands user paths on Linux, OSX, and windows"""

    os_name = os_name.lower()
    assert os_name in ['windows', 'osx', 'linux']

    if not isinstance(paths, list):
        paths = [paths]

    if os_name == 'windows':
        paths = map(_expand_win_path, paths)
    else:
        paths = map(os.path.expanduser, paths)

    for path in paths:
        for i in sorted(glob.glob(path)):   # glob will return results in arbitrary order. sorted() is use to make output predictable.
            yield i                         # can use return here without using `_expand_paths()` below.
                                            # but using generator can be useful if we plan to parse all `Cookies` files later.

def _expand_paths(paths:list, os_name:str):
    return next(_expand_paths_impl(paths, os_name), None)

def _normalize_genarate_paths_chromium(paths:Union[str,list], channel:Union[str,list]=None):
    channel = channel or ['']
    if not isinstance(channel, list):
        channel = [channel]
    if not isinstance(paths, list):
        paths = [paths]
    return paths, channel

def _genarate_nix_paths_chromium(paths:Union[str,list], channel:Union[str,list]=None):
    """Generate paths for chromium based browsers on *nix systems."""

    paths, channel = _normalize_genarate_paths_chromium(paths, channel)
    genararated_paths = []
    for chan in channel:
        for path in paths:
            genararated_paths.append(path.format(channel = chan))
    return genararated_paths

def _genarate_win_paths_chromium(paths:Union[str,list], channel:Union[str,list]=None):
    """Generate paths for chromium based browsers on windows"""

    paths, channel = _normalize_genarate_paths_chromium(paths, channel)
    genararated_paths = []
    for chan in channel:
        for path in paths:
            genararated_paths.append({'env': 'APPDATA', 'path': '..\\Local\\' + path.format(channel = chan)})
            genararated_paths.append({'env': 'LOCALAPPDATA', 'path': path.format(channel = chan)})
            genararated_paths.append({'env': 'APPDATA', 'path': path.format(channel = chan)})
    return genararated_paths

def _text_factory(data):
    try:
        return data.decode('utf-8')
    except UnicodeDecodeError:
        return data

class _JeepneyConnection:
    def __init__(self, object_path, bus_name, interface):
        self.__dbus_address = jeepney.DBusAddress(object_path, bus_name, interface)

    def __enter__(self):
        self.__connection = open_dbus_connection()
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        self.__connection.close()

    def close(self):
        self.__connection.close()

    def call_method(self, method_name, signature=None, *args):
        method = jeepney.new_method_call(self.__dbus_address, method_name, signature, args)
        response = self.__connection.send_and_get_reply(method)
        if response.header.message_type == jeepney.MessageType.error:
            raise RuntimeError(response.body[0])
        return response.body[0] if len(response.body) == 1 else response.body

class _LinuxPasswordManager:
    """Retrieve password used to encrypt cookies from KDE Wallet or SecretService"""

    _APP_ID = 'browser-cookie3'
    def __init__(self, use_dbus):
        if use_dbus:
            self.__methods_map = {
                'kwallet' : self.__get_kdewallet_password_dbus,
                'secretstorage': self.__get_secretstorage_item_dbus
            }
        else:
            self.__methods_map = {
                'kwallet' : self.__get_kdewallet_password_jeepney,
                'secretstorage': self.__get_secretstorage_item_jeepney
            }

    def get_password(self, os_crypt_name):
        try:
            return self.__get_secretstorage_password(os_crypt_name)
        except RuntimeError:
            pass
        try:
            return self.__methods_map.get('kwallet')(os_crypt_name)
        except RuntimeError:
            pass
        # try default peanuts password, probably won't work
        return CHROMIUM_DEFAULT_PASSWORD

    def __get_secretstorage_password(self, os_crypt_name):
        schemas = ['chrome_libsecret_os_crypt_password_v2', 'chrome_libsecret_os_crypt_password_v1']
        for schema in schemas:
            try:
                return self.__methods_map.get('secretstorage')(schema, os_crypt_name)
            except RuntimeError:
                pass
        raise RuntimeError(f'Can not find secret for {os_crypt_name}')

    def __get_secretstorage_item_dbus(self, schema: str, application: str):
        with contextlib.closing(dbus.SessionBus()) as connection:
            try:
                secret_service = dbus.Interface(
                    connection.get_object('org.freedesktop.secrets', '/org/freedesktop/secrets', False),
                    'org.freedesktop.Secret.Service',
                )
            except dbus.exceptions.DBusException:
                raise RuntimeError("The name org.freedesktop.secrets was not provided by any .service files")
            object_path = secret_service.SearchItems({
                'xdg:schema': schema,
                'application': application,
            })
            object_path = list(filter(lambda x: len(x), object_path))
            if len(object_path) == 0:
                raise RuntimeError(f'Can not find secret for {application}')
            object_path = object_path[0][0]

            secret_service.Unlock([object_path])
            _, session = secret_service.OpenSession('plain', dbus.String('', variant_level=1))
            _, _, secret, _ = secret_service.GetSecrets([object_path], session)[object_path]
            return bytes(secret)

    def __get_kdewallet_password_dbus(self, os_crypt_name):
        folder = f'{os_crypt_name.capitalize()} Keys'
        key = f'{os_crypt_name.capitalize()} Safe Storage'
        with contextlib.closing(dbus.SessionBus()) as connection:
            try:
                kwalletd5_object = connection.get_object('org.kde.kwalletd5', '/modules/kwalletd5', False)
            except dbus.exceptions.DBusException:
                raise RuntimeError("The name org.kde.kwalletd5 was not provided by any .service files")
            kwalletd5 = dbus.Interface(kwalletd5_object, 'org.kde.KWallet')
            handle = kwalletd5.open(kwalletd5.networkWallet(), dbus.Int64(0), self._APP_ID)
            if not kwalletd5.hasFolder(handle, folder, self._APP_ID):
                kwalletd5.close(handle, False, self._APP_ID)
                raise RuntimeError(f'KDE Wallet folder {folder} not found.')
            password = kwalletd5.readPassword(handle, folder, key, self._APP_ID)
            kwalletd5.close(handle, False, self._APP_ID)
            return password.encode('utf-8')

    def __get_secretstorage_item_jeepney(self, schema, application):
        with _JeepneyConnection('/org/freedesktop/secrets', 'org.freedesktop.secrets', 'org.freedesktop.Secret.Service') as connection:
            object_path = connection.call_method('SearchItems', 'a{ss}', {'xdg:schema': schema, 'application': application})
            object_path = list(filter(lambda x: len(x), object_path))
            if len(object_path) == 0:
                raise RuntimeError(f'Can not find secret for {application}')
            object_path = object_path[0][0]
            connection.call_method('Unlock', 'ao', [object_path])
            _, session = connection.call_method('OpenSession', 'sv', 'plain', ('s', ''))
            _, _, secret, _ = connection.call_method('GetSecrets', 'aoo', [object_path], session)[object_path]
            return secret

    def __get_kdewallet_password_jeepney(self, folder, key):
        with _JeepneyConnection('/modules/kwalletd5', 'org.kde.kwalletd5', 'org.kde.KWallet') as connection:
            network_wallet = connection.call_method('networkWallet')
            handle = connection.call_method('open', 'sxs', network_wallet, 0, self._APP_ID)
            has_folder = connection.call_method('hasFolder', 'iss', handle, folder, self._APP_ID)
            if not has_folder:
                connection.call_method('close', 'ibs', handle, False, self._APP_ID)
                raise RuntimeError(f'KDE Wallet folder {folder} not found.')
            password = connection.call_method('readPassword', 'isss', handle, folder, key, self._APP_ID)
            connection.call_method('close', 'ibs', handle, False, self._APP_ID)
            return password.encode('utf-8')

class ChromiumBased:
    """Super class for all Chromium based browsers"""

    UNIX_TO_NT_EPOCH_OFFSET = 11644473600  # seconds from 1601-01-01T00:00:00Z to 1970-01-01T00:00:00Z

    def __init__(self, browser:str, cookie_file=None, domain_name="", key_file=None, **kwargs):
        self.salt = b'saltysalt'
        self.iv = b' ' * 16
        self.length = 16
        self.browser = browser
        self.cookie_file = cookie_file
        self.domain_name = domain_name
        self.key_file = key_file
        self.__add_key_and_cookie_file(**kwargs)

    def __add_key_and_cookie_file(self,
            linux_cookies=None, windows_cookies=None, osx_cookies=None,
            windows_keys=None, os_crypt_name=None, osx_key_service=None, osx_key_user=None):

        if sys.platform == 'darwin':
            password = _get_osx_keychain_password(osx_key_service, osx_key_user)
            iterations = 1003  # number of pbkdf2 iterations on mac
            self.v10_key = PBKDF2(password, self.salt, self.length, iterations)
            cookie_file = self.cookie_file or _expand_paths(osx_cookies,'osx')

        elif sys.platform.startswith('linux') or 'bsd' in sys.platform.lower():
            password = _LinuxPasswordManager(USE_DBUS_LINUX).get_password(os_crypt_name)
            iterations = 1
            self.v10_key = PBKDF2(CHROMIUM_DEFAULT_PASSWORD, self.salt, self.length, iterations)
            self.v11_key = PBKDF2(password, self.salt, self.length, iterations)

            cookie_file = self.cookie_file or _expand_paths(linux_cookies, 'linux')

        elif sys.platform == "win32":
            key_file = self.key_file or _expand_paths(windows_keys,'windows')

            if key_file:
                with open(key_file,'rb') as f:
                    key_file_json = json.load(f)
                    key64 = key_file_json['os_crypt']['encrypted_key'].encode('utf-8')

                    # Decode Key, get rid of DPAPI prefix, unprotect data
                    keydpapi = base64.standard_b64decode(key64)[5:]
                    _, self.v10_key = _crypt_unprotect_data(keydpapi, is_key=False)

            # get cookie file from APPDATA

            cookie_file = self.cookie_file

            if not cookie_file:
                if self.browser.lower() == 'chrome' and _windows_group_policy_path():
                    cookie_file = _windows_group_policy_path()
                else:
                    cookie_file = _expand_paths(windows_cookies,'windows')

        else:
            raise BrowserCookieError(
                "OS not recognized. Works on OSX, Windows, and Linux.")

        if not cookie_file:
            raise BrowserCookieError('Failed to find {} cookie'.format(self.browser))

        self.tmp_cookie_file = _create_local_copy(cookie_file)

    def __del__(self):
        # remove temporary backup of sqlite cookie database
        if hasattr(self, 'tmp_cookie_file'):  # if there was an error till here
            os.remove(self.tmp_cookie_file)

    def __str__(self):
        return self.browser

    def load(self):
        """Load sqlite cookies into a cookiejar"""
        con = sqlite3.connect(self.tmp_cookie_file)
        con.text_factory = _text_factory
        cur = con.cursor()
        try:
            # chrome <=55
            cur.execute('SELECT host_key, path, secure, expires_utc, name, value, encrypted_value, is_httponly '
                        'FROM cookies WHERE host_key like ?;', ('%{}%'.format(self.domain_name),))
        except sqlite3.OperationalError:
            # chrome >=56
            cur.execute('SELECT host_key, path, is_secure, expires_utc, name, value, encrypted_value, is_httponly '
                        'FROM cookies WHERE host_key like ?;', ('%{}%'.format(self.domain_name),))

        cj = http.cookiejar.CookieJar()

        for item in cur.fetchall():
            # Per https://github.com/chromium/chromium/blob/main/base/time/time.h#L5-L7,
            # Chromium-based browsers store cookies' expiration timestamps as MICROSECONDS elapsed
            # since the Windows NT epoch (1601-01-01 0:00:00 GMT), or 0 for session cookies.
            #
            # http.cookiejar stores cookies' expiration timestamps as SECONDS since the Unix epoch
            # (1970-01-01 0:00:00 GMT, or None for session cookies.
            host, path, secure, expires_nt_time_epoch, name, value, enc_value, http_only = item
            if (expires_nt_time_epoch == 0):
                expires = None
            else:
                expires = (expires_nt_time_epoch / 1000000) - self.UNIX_TO_NT_EPOCH_OFFSET

            value = self._decrypt(value, enc_value)
            c = create_cookie(host, path, secure, expires, name, value, http_only)
            cj.set_cookie(c)
        con.close()
        return cj

    @staticmethod
    def _decrypt_windows_chromium(value, encrypted_value):

        if len(value) != 0:
            return value

        if encrypted_value == "":
            return ""

        _, data = _crypt_unprotect_data(encrypted_value)
        assert isinstance(data, bytes)
        return data.decode()

    def _decrypt(self, value, encrypted_value):
        """Decrypt encoded cookies"""

        if sys.platform == 'win32':
            try:
                return self._decrypt_windows_chromium(value, encrypted_value)

            # Fix for change in Chrome 80
            except RuntimeError:  # Failed to decrypt the cipher text with DPAPI
                if not self.v10_key:
                    raise RuntimeError(
                        'Failed to decrypt the cipher text with DPAPI and no AES key.')
                # Encrypted cookies should be prefixed with 'v10' according to the
                # Chromium code. Strip it off.
                encrypted_value = encrypted_value[3:]
                nonce, tag = encrypted_value[:12], encrypted_value[-16:]
                aes = AES.new(self.v10_key, AES.MODE_GCM, nonce=nonce)

                # will rise Value Error: MAC check failed byte if the key is wrong,
                # probably we did not got the key and used peanuts
                try:
                    data = aes.decrypt_and_verify(encrypted_value[12:-16], tag)
                except ValueError:
                    raise BrowserCookieError('Unable to get key for cookie decryption')
                return data.decode()

        if value or (encrypted_value[:3] not in [b'v11', b'v10']):
            return value

        # Encrypted cookies should be prefixed with 'v10' on mac,
        # 'v10' or 'v11' on Linux. Choose key based on this prefix.
        # Reference in chromium code: `OSCryptImpl::DecryptString` in
        # components/os_crypt/os_crypt_linux.cc
        if not hasattr(self, 'v11_key'):
            assert encrypted_value[:3] != b'v11', "v11 keys should only appear on Linux."
        key = self.v11_key if encrypted_value[:3] == b'v11' else self.v10_key
        encrypted_value = encrypted_value[3:]
        cipher = AES.new(key, AES.MODE_CBC, self.iv)

        # will rise Value Error: invalid padding byte if the key is wrong,
        # probably we did not got the key and used peanuts
        try:
            decrypted = unpad(cipher.decrypt(encrypted_value), AES.block_size)
        except ValueError:
            raise BrowserCookieError('Unable to get key for cookie decryption')
        return decrypted.decode('utf-8')

class Chrome(ChromiumBased):
    """Class for Google Chrome"""
    def __init__(self, cookie_file=None, domain_name="", key_file=None):
        args = {
            'linux_cookies': _genarate_nix_paths_chromium(
                [
                    '~/.config/google-chrome{channel}/Default/Cookies',
                    '~/.config/google-chrome{channel}/Profile */Cookies'
                ],
                channel=['', '-beta', '-unstable']
            ),
            'windows_cookies': _genarate_win_paths_chromium(
                [
                    'Google\\Chrome{channel}\\User Data\\Default\\Cookies',
                    'Google\\Chrome{channel}\\User Data\\Default\\Network\\Cookies',
                    'Google\\Chrome{channel}\\User Data\\Profile *\\Cookies',
                    'Google\\Chrome{channel}\\User Data\\Profile *\\Network\\Cookies'
                ],
                channel=['', ' Beta', ' Dev']
            ),
            'osx_cookies': _genarate_nix_paths_chromium(
                [
                    '~/Library/Application Support/Google/Chrome{channel}/Default/Cookies',
                    '~/Library/Application Support/Google/Chrome{channel}/Profile */Cookies'
                ],
                channel=['', ' Beta', ' Dev']
            ),
            'windows_keys': _genarate_win_paths_chromium(
                'Google\\Chrome{channel}\\User Data\\Local State',
                channel=['', ' Beta', ' Dev']
            ),
            'os_crypt_name': 'chrome',
            'osx_key_service' : 'Chrome Safe Storage',
            'osx_key_user' : 'Chrome'
        }
        super().__init__(browser='Chrome', cookie_file=cookie_file, domain_name=domain_name, key_file=key_file, **args)

class Chromium(ChromiumBased):
    """Class for Chromium"""
    def __init__(self, cookie_file=None, domain_name="", key_file=None):
        args = {
            'linux_cookies':[
                '~/.config/chromium/Default/Cookies',
                '~/.config/chromium/Profile */Cookies'
            ],
            'windows_cookies': _genarate_win_paths_chromium(
                [
                    'Chromium\\User Data\\Default\\Cookies',
                    'Chromium\\User Data\\Default\\Network\\Cookies',
                    'Chromium\\User Data\\Profile *\\Cookies',
                    'Chromium\\User Data\\Profile *\\Network\\Cookies'
                ]
            ),
            'osx_cookies': [
                '~/Library/Application Support/Chromium/Default/Cookies',
                '~/Library/Application Support/Chromium/Profile */Cookies'
            ],
            'windows_keys': _genarate_win_paths_chromium(
                'Chromium\\User Data\\Local State'
            ),
            'os_crypt_name': 'chromium',
            'osx_key_service' : 'Chromium Safe Storage',
            'osx_key_user' : 'Chromium'
        }
        super().__init__(browser='Chromium', cookie_file=cookie_file, domain_name=domain_name, key_file=key_file, **args)

class Opera(ChromiumBased):
    """Class for Opera"""
    def __init__(self, cookie_file=None, domain_name="", key_file=None):
        args = {
            'linux_cookies': [
                '~/.config/opera/Cookies',
                '~/.config/opera-beta/Cookies',
                '~/.config/opera-developer/Cookies'
            ],
            'windows_cookies': _genarate_win_paths_chromium(
                [
                    'Opera Software\\Opera {channel}\\Cookies',
                    'Opera Software\\Opera {channel}\\Network\\Cookies'
                ],
                channel=['Stable', 'Next', 'Developer']
            ),
            'osx_cookies': [
                '~/Library/Application Support/com.operasoftware.Opera/Cookies',
                '~/Library/Application Support/com.operasoftware.OperaNext/Cookies',
                '~/Library/Application Support/com.operasoftware.OperaDeveloper/Cookies'
            ],
            'windows_keys': _genarate_win_paths_chromium(
                'Opera Software\\Opera {channel}\\Local State',
                channel=['Stable', 'Next', 'Developer']
            ),
            'os_crypt_name': 'chromium',
            'osx_key_service' : 'Opera Safe Storage',
            'osx_key_user' : 'Opera'
        }
        super().__init__(browser='Opera', cookie_file=cookie_file, domain_name=domain_name, key_file=key_file, **args)

class OperaGX(ChromiumBased):
    """Class for Opera GX"""
    def __init__(self, cookie_file=None, domain_name="", key_file=None):
        args = {
            'linux_cookies': [], # Not available on Linux
            'windows_cookies': _genarate_win_paths_chromium(
                [
                    'Opera Software\\Opera GX {channel}\\Cookies',
                    'Opera Software\\Opera GX {channel}\\Network\\Cookies'
                ],
                channel=['Stable']
            ),
            'osx_cookies': ['~/Library/Application Support/com.operasoftware.OperaGX/Cookies'],
            'windows_keys': _genarate_win_paths_chromium(
                'Opera Software\\Opera GX {channel}\\Local State',
                channel=['Stable']
            ),
            'os_crypt_name': 'chromium',
            'osx_key_service' : 'Opera Safe Storage',
            'osx_key_user' : 'Opera'
        }
        super().__init__(browser='Opera GX', cookie_file=cookie_file, domain_name=domain_name, key_file=key_file, **args)

class Brave(ChromiumBased):
    def __init__(self, cookie_file=None, domain_name="", key_file=None):
        args = {
            'linux_cookies':_genarate_nix_paths_chromium(
                [
                    '~/.config/BraveSoftware/Brave-Browser{channel}/Default/Cookies',
                    '~/.config/BraveSoftware/Brave-Browser{channel}/Profile */Cookies'
                ],
                channel=['', '-Beta', '-Dev', '-Nightly']
            ),
            'windows_cookies': _genarate_win_paths_chromium(
                [
                    'BraveSoftware\\Brave-Browser{channel}\\User Data\\Default\\Cookies',
                    'BraveSoftware\\Brave-Browser{channel}\\User Data\\Default\\Network\\Cookies',
                    'BraveSoftware\\Brave-Browser{channel}\\User Data\\Profile *\\Cookies',
                    'BraveSoftware\\Brave-Browser{channel}\\User Data\\Profile *\\Network\\Cookies'
                ],
                channel=['', '-Beta', '-Dev', '-Nightly']
            ),
            'osx_cookies': _genarate_nix_paths_chromium(
                [
                    '~/Library/Application Support/BraveSoftware/Brave-Browser{channel}/Default/Cookies',
                    '~/Library/Application Support/BraveSoftware/Brave-Browser{channel}/Profile */Cookies'
                ],
                channel=['', '-Beta', '-Dev', '-Nightly']
            ),
            'windows_keys': _genarate_win_paths_chromium(
                'BraveSoftware\\Brave-Browser{channel}\\User Data\\Local State',
                channel=['', '-Beta', '-Dev', '-Nightly']
            ),
            'os_crypt_name': 'brave',
            'osx_key_service' : 'Brave Safe Storage',
            'osx_key_user' : 'Brave'
        }
        super().__init__(browser='Brave', cookie_file=cookie_file, domain_name=domain_name, key_file=key_file, **args)

class Edge(ChromiumBased):
    """Class for Microsoft Edge"""
    def __init__(self, cookie_file=None, domain_name="", key_file=None):
        args = {
            'linux_cookies': _genarate_nix_paths_chromium(
                [
                    '~/.config/microsoft-edge{channel}/Default/Cookies',
                    '~/.config/microsoft-edge{channel}/Profile */Cookies'
                ],
                channel=['', '-beta', '-dev']
            ),
            'windows_cookies': _genarate_win_paths_chromium(
                [
                    'Microsoft\\Edge{channel}\\User Data\\Default\\Cookies',
                    'Microsoft\\Edge{channel}\\User Data\\Default\\Network\\Cookies',
                    'Microsoft\\Edge{channel}\\User Data\\Profile *\\Cookies',
                    'Microsoft\\Edge{channel}\\User Data\\Profile *\\Network\\Cookies'
                ],
                channel=['', ' Beta', ' Dev', ' SxS']
            ),
            'osx_cookies': _genarate_nix_paths_chromium(
                [
                    '~/Library/Application Support/Microsoft Edge{channel}/Default/Cookies',
                    '~/Library/Application Support/Microsoft Edge{channel}/Profile */Cookies'
                ],
                channel=['', ' Beta', ' Dev', ' Canary']
            ),
            'windows_keys': _genarate_win_paths_chromium(
                'Microsoft\\Edge{channel}\\User Data\\Local State',
                channel=['', ' Beta', ' Dev', ' SxS']
            ),
            'os_crypt_name': 'chromium',
            'osx_key_service' : 'Microsoft Edge Safe Storage',
            'osx_key_user' : 'Microsoft Edge'
        }
        super().__init__(browser='Edge', cookie_file=cookie_file, domain_name=domain_name, key_file=key_file, **args)

class Vivaldi(ChromiumBased):
    """Class for Vivaldi Browser"""
    def __init__(self, cookie_file=None, domain_name="", key_file=None):
        args = {
            'linux_cookies': [
                '~/.config/vivaldi/Default/Cookies',
                '~/.config/vivaldi/Profile */Cookies',
                '~/.config/vivaldi-snapshot/Default/Cookies',
                '~/.config/vivaldi-snapshot/Profile */Cookies'
            ],
            'windows_cookies': _genarate_win_paths_chromium(
                [
                    'Vivaldi\\User Data\\Default\\Cookies',
                    'Vivaldi\\User Data\\Default\\Network\\Cookies',
                    'Vivaldi\\User Data\\Profile *\\Cookies',
                    'Vivaldi\\User Data\\Profile *\\Network\\Cookies'
                ]
            ),
            'osx_cookies': [
                '~/Library/Application Support/Vivaldi/Default/Cookies',
                '~/Library/Application Support/Vivaldi/Profile */Cookies'
            ],
            'windows_keys': _genarate_win_paths_chromium(
                'Vivaldi\\User Data\\Local State'
            ),
            'os_crypt_name': 'chrome',
            'osx_key_service' : 'Vivaldi Safe Storage',
            'osx_key_user' : 'Vivaldi'
        }
        super().__init__(browser='Vivaldi', cookie_file=cookie_file, domain_name=domain_name, key_file=key_file, **args)

class Firefox:
    """Class for Firefox"""
    def __init__(self, cookie_file=None, domain_name=""):
        self.tmp_cookie_file = None
        cookie_file = cookie_file or self.find_cookie_file()
        self.tmp_cookie_file = _create_local_copy(cookie_file)
        # current sessions are saved in sessionstore.js
        self.session_file = os.path.join(
            os.path.dirname(cookie_file), 'sessionstore.js')
        self.session_file_lz4 = os.path.join(os.path.dirname(
            cookie_file), 'sessionstore-backups', 'recovery.jsonlz4')
        # domain name to filter cookies by
        self.domain_name = domain_name

    def __del__(self):
        # remove temporary backup of sqlite cookie database
        if self.tmp_cookie_file:
            os.remove(self.tmp_cookie_file)

    def __str__(self):
        return 'firefox'

    @staticmethod
    def get_default_profile(user_data_path):
        config = configparser.ConfigParser()
        profiles_ini_path = glob.glob(os.path.join(
            user_data_path + '**', 'profiles.ini'))
        fallback_path = user_data_path + '**'

        if not profiles_ini_path:
            return fallback_path

        profiles_ini_path = profiles_ini_path[0]
        config.read(profiles_ini_path, encoding="utf8")

        profile_path = None
        for section in config.sections():
            if section.startswith('Install'):
                profile_path = config[section].get('Default')
                break
            # in ff 72.0.1, if both an Install section and one with Default=1 are present, the former takes precedence
            elif config[section].get('Default') == '1' and not profile_path:
                profile_path = config[section].get('Path')

        for section in config.sections():
            # the Install section has no relative/absolute info, so check the profiles
            if config[section].get('Path') == profile_path:
                absolute = config[section].get('IsRelative') == '0'
                return profile_path if absolute else os.path.join(os.path.dirname(profiles_ini_path), profile_path)

        return fallback_path

    @staticmethod
    def find_cookie_file():
        cookie_files = []

        if sys.platform == 'darwin':
            user_data_path = os.path.expanduser(
                '~/Library/Application Support/Firefox')
        elif sys.platform.startswith('linux') or 'bsd' in sys.platform.lower():
            # Looking for cookies from a Snap based Firefox first, as some
            # users might have profiles at both this and the other location,
            # as they were migrated to Snap by their OS at some point, leaving
            # cookies at the other location outdated.
            general_path = os.path.expanduser('~/snap/firefox/common/.mozilla/firefox')
            if os.path.isdir(general_path):
                user_data_path = general_path
            else:
                user_data_path = os.path.expanduser('~/.mozilla/firefox')
        elif sys.platform == 'win32':
            user_data_path = os.path.join(
                os.environ.get('APPDATA'), 'Mozilla', 'Firefox')
            # legacy firefox <68 fallback
            cookie_files = glob.glob(os.path.join(os.environ.get('PROGRAMFILES'), 'Mozilla Firefox', 'profile', 'cookies.sqlite')) \
                or glob.glob(os.path.join(os.environ.get('PROGRAMFILES(X86)'), 'Mozilla Firefox', 'profile', 'cookies.sqlite'))
        else:
            raise BrowserCookieError(
                'Unsupported operating system: ' + sys.platform)

        cookie_files = glob.glob(os.path.join(Firefox.get_default_profile(user_data_path), 'cookies.sqlite')) \
            or cookie_files

        if cookie_files:
            return cookie_files[0]
        else:
            raise BrowserCookieError('Failed to find Firefox cookie file')

    @staticmethod
    def __create_session_cookie(cookie_json):
        return create_cookie(cookie_json.get('host', ''), cookie_json.get('path', ''),
                             cookie_json.get('secure', False), None,
                             cookie_json.get('name', ''), cookie_json.get('value', ''),
                             cookie_json.get('httponly', False))

    def __add_session_cookies(self, cj):
        if not os.path.exists(self.session_file):
            return
        try:
            json_data = json.loads(
                open(self.session_file, 'rb').read().decode())
        except ValueError as e:
            print('Error parsing firefox session JSON:', str(e))
        else:
            for window in json_data.get('windows', []):
                for cookie in window.get('cookies', []):
                    if self.domain_name == '' or self.domain_name in cookie.get('host', ''):
                        cj.set_cookie(Firefox.__create_session_cookie(cookie))

    def __add_session_cookies_lz4(self, cj):
        if not os.path.exists(self.session_file_lz4):
            return
        try:
            file_obj = open(self.session_file_lz4, 'rb')
            file_obj.read(8)
            json_data = json.loads(lz4.block.decompress(file_obj.read()))
        except ValueError as e:
            print('Error parsing firefox session JSON LZ4:', str(e))
        else:
            for cookie in json_data.get('cookies', []):
                if self.domain_name == '' or self.domain_name in cookie.get('host', ''):
                    cj.set_cookie(Firefox.__create_session_cookie(cookie))

    def load(self):
        con = sqlite3.connect(self.tmp_cookie_file)
        cur = con.cursor()
        cur.execute('select host, path, isSecure, expiry, name, value, isHttpOnly from moz_cookies '
                    'where host like ?', ('%{}%'.format(self.domain_name),))

        cj = http.cookiejar.CookieJar()
        for item in cur.fetchall():
            host, path, secure, expires, name, value, http_only = item
            c = create_cookie(host, path, secure, expires, name, value, http_only)
            cj.set_cookie(c)
        con.close()

        self.__add_session_cookies(cj)
        self.__add_session_cookies_lz4(cj)

        return cj

class Safari:
    """Class for Safari"""

    APPLE_TO_UNIX_TIME = 978307200
    NEW_ISSUE_MESSAGE = 'Page format changed.\nPlease create a new issue on: https://github.com/borisbabic/browser_cookie3/issues/new'
    safari_cookies = [
        '~/Library/Containers/com.apple.Safari/Data/Library/Cookies/Cookies.binarycookies',
        '~/Library/Cookies/Cookies.binarycookies'
    ]

    def __init__(self, cookie_file=None, domain_name="") -> None:
        self.__offset = 0
        self.__domain_name = domain_name
        self.__buffer = None
        self.__open_file(cookie_file)
        self.__parse_header()

    def __del__(self):
        if self.__buffer:
            self.__buffer.close()

    def __open_file(self, cookie_file):
        cookie_file = cookie_file or _expand_paths(self.safari_cookies, 'osx')
        if not cookie_file:
            raise BrowserCookieError('Can not find Safari cookie file')
        self.__buffer = open(cookie_file, 'rb')

    def __read_file(self, size:int, offset:int=None):
        if offset is not None:
            self.__offset = offset
        self.__buffer.seek(self.__offset)
        self.__offset += size
        return BytesIO(self.__buffer.read(size))

    def __parse_header(self):
        assert self.__buffer.read(4) == b'cook', 'Not a safari cookie file'
        self.__total_page = struct.unpack('>I', self.__buffer.read(4))[0]

        self.__page_sizes = []
        for _ in range(self.__total_page):
            self.__page_sizes.append(struct.unpack('>I', self.__buffer.read(4))[0])

    @staticmethod
    def __read_until_null(file:BytesIO, decode:bool=True):
        data = []
        while True:
            byte = file.read(1)
            if byte == b'\x00':
                break
            data.append(byte)
        data = b''.join(data)
        if decode:
            data = data.decode('utf-8')
        return data

    def __parse_cookie(self, page:BytesIO, cookie_offset:int):
        page.seek(cookie_offset)
        cookie_size = struct.unpack('<I', page.read(4))[0]
        page.seek(4, 1) # skip 4-bytes unknown data
        flags = struct.unpack('<I', page.read(4))[0]
        page.seek(4, 1) # skip 4-bytes unknown data
        is_secure = bool(flags & 0x1)
        is_httponly = bool(flags & 0x4)

        host_offset = struct.unpack('<I', page.read(4))[0]
        name_offset = struct.unpack('<I', page.read(4))[0]
        path_offset = struct.unpack('<I', page.read(4))[0]
        value_offset = struct.unpack('<I', page.read(4))[0]
        comment_offset = struct.unpack('<I', page.read(4))[0]

        assert page.read(4) == b'\x00\x00\x00\x00', self.NEW_ISSUE_MESSAGE
        expiry_date = int(struct.unpack('<d', page.read(8))[0] + self.APPLE_TO_UNIX_TIME) # convert to unix time
        creation_time = int(struct.unpack('<d', page.read(8))[0] + self.APPLE_TO_UNIX_TIME) # convert to unix time

        page.seek(cookie_offset + host_offset, 0)
        host = self.__read_until_null(page)
        page.seek(cookie_offset + name_offset, 0)
        name = self.__read_until_null(page)
        page.seek(cookie_offset + path_offset, 0)
        path = self.__read_until_null(page)
        page.seek(cookie_offset + value_offset, 0)
        value = self.__read_until_null(page)
        if comment_offset:
            page.seek(cookie_offset + comment_offset, 0)
            comment = self.__read_until_null(page)

        return create_cookie(host, path, is_secure, expiry_date, name, value, is_httponly)

    def __domain_filter(self, cookie: http.cookiejar.Cookie):
        if not self.__domain_name:
            return True
        return self.__domain_name in cookie.domain

    def __parse_page(self, page_index:int):
        offset = 8 + self.__total_page * 4 + sum(self.__page_sizes[:page_index])
        page = self.__read_file(self.__page_sizes[page_index], offset)
        assert page.read(4) == b'\x00\x00\x01\x00', self.NEW_ISSUE_MESSAGE
        n_cookies = struct.unpack('<I', page.read(4))[0]
        cookie_offsets = []
        for _ in range(n_cookies):
            cookie_offsets.append(struct.unpack('<I', page.read(4))[0])
        assert page.read(4) == b'\x00\x00\x00\x00', self.NEW_ISSUE_MESSAGE

        for offset in cookie_offsets:
            yield self.__parse_cookie(page, offset)

    def load(self):
        cj = http.cookiejar.CookieJar()
        for i in range(self.__total_page):
            for cookie in self.__parse_page(i):
                if self.__domain_filter(cookie):
                    cj.set_cookie(cookie)
        return cj

def create_cookie(host, path, secure, expires, name, value, http_only):
    """Shortcut function to create a cookie"""
    # HTTPOnly flag goes in _rest, if present (see https://github.com/python/cpython/pull/17471/files#r511187060)
    return http.cookiejar.Cookie(0, name, value, None, False, host, host.startswith('.'), host.startswith('.'), path,
                                 True, secure, expires, False, None, None,
                                 {'HTTPOnly': ''} if http_only else {})

def chrome(cookie_file=None, domain_name="", key_file=None):
    """Returns a cookiejar of the cookies used by Chrome. Optionally pass in a
    domain name to only load cookies from the specified domain
    """
    return Chrome(cookie_file, domain_name, key_file).load()

def chromium(cookie_file=None, domain_name="", key_file=None):
    """Returns a cookiejar of the cookies used by Chromium. Optionally pass in a
    domain name to only load cookies from the specified domain
    """
    return Chromium(cookie_file, domain_name, key_file).load()

def opera(cookie_file=None, domain_name="", key_file=None):
    """Returns a cookiejar of the cookies used by Opera. Optionally pass in a
    domain name to only load cookies from the specified domain
    """
    return Opera(cookie_file, domain_name, key_file).load()

def opera_gx(cookie_file=None, domain_name="", key_file=None):
    """Returns a cookiejar of the cookies used by Opera GX. Optionally pass in a
    domain name to only load cookies from the specified domain
    """
    return OperaGX(cookie_file, domain_name, key_file).load()

def brave(cookie_file=None, domain_name="", key_file=None):
    """Returns a cookiejar of the cookies and sessions used by Brave. Optionally
    pass in a domain name to only load cookies from the specified domain
    """
    return Brave(cookie_file, domain_name, key_file).load()

def edge(cookie_file=None, domain_name="", key_file=None):
    """Returns a cookiejar of the cookies used by Microsoft Egde. Optionally pass in a
    domain name to only load cookies from the specified domain
    """
    return Edge(cookie_file, domain_name, key_file).load()

def vivaldi(cookie_file=None, domain_name="", key_file=None):
    """Returns a cookiejar of the cookies used by Vivaldi Browser. Optionally pass in a
    domain name to only load cookies from the specified domain
    """
    return Vivaldi(cookie_file, domain_name, key_file).load()

def firefox(cookie_file=None, domain_name=""):
    """Returns a cookiejar of the cookies and sessions used by Firefox. Optionally
    pass in a domain name to only load cookies from the specified domain
    """
    return Firefox(cookie_file, domain_name).load()

def safari(cookie_file=None, domain_name=""):
    """Returns a cookiejar of the cookies and sessions used by Safari. Optionally
    pass in a domain name to only load cookies from the specified domain
    """
    return Safari(cookie_file, domain_name).load()

def load(domain_name=""):
    """Try to load cookies from all supported browsers and return combined cookiejar
    Optionally pass in a domain name to only load cookies from the specified domain
    """
    cj = http.cookiejar.CookieJar()
    for cookie_fn in [chrome, chromium, opera, opera_gx, brave, edge, vivaldi, firefox, safari]:
        try:
            for cookie in cookie_fn(domain_name=domain_name):
                cj.set_cookie(cookie)
        except BrowserCookieError:
            pass
    return cj

if __name__ == '__main__':
    print(load())