text:
import os
import re
from os.path import join
from itertools import count

import logbook

from .utils import CachedAttribute
from .schemes import Enum

logger = logbook.Logger()

class Token(object):

    def __init__(self, enum, text, linedata):
        self.enum = enum
        self.text = text
        self.linedata = linedata

    def __iter__(self):
        for x in (self.enum, self.text, self.linedata):
            yield x

    def __repr__(self):
        return 'Token(%r, %r, %r)' % (self.enum, self.text, self.linedata)

    @classmethod
    def make(cls, tpl):
        return cls(*tpl)

class Stream(object):

    def __init__(self, iterable):
        self._stream = map(Token.make, iterable)
        self.i = 0

    def __iter__(self):
        self.i = -1
        while True:
            self.i += 1
            try:
                yield self._stream[self.i]
            except IndexError:
                raise StopIteration

    def next(self):
        i = self.i + 1
        try:
            yield self._stream[i]
        except IndexError:
            raise StopIteration

    def previous(self):
        return self.behind(1)

    def this(self):
        return self._stream[self.i]

    def ahead(self, n):
        return self._stream[self.i + n]

    def behind(self, n):
        return self._stream[self.i - n]

class BaseNode(list):

    def filesystem_dump(self, path, root=True):
        text_counter = count()
        for node in self:

            if isinstance(node, TextNode):

                # Unenumerate, single child (a.k.a, top-level section text).
                filename = 'text' + str(next(text_counter))
                filename = join(path, filename)
                logger.info('Writing text to %s' % filename)
                with open(filename, 'w') as f:
                    text = node.content.encode('utf-8')
                    logger.info('..text: ' + repr(text[:70] + '...'))
                    f.write(text)

            else:
                if node.enum:
                    newdir = join(path, str(node.enum.text))
                    os.mkdir(newdir)

            node.filesystem_dump(path, root=False)

class TextNode(BaseNode):
    '''A text node can (sometimes) have children.'''

    def __init__(self, content):
        self.content = content

    def __repr__(self):
        return 'TextNode(%r)' % self.content

    def json(self):
        return {'type': 'textnode',
                'content': self.content,
                'sub': [node.json() for node in self]}

class Node(BaseNode):

    def __init__(self, enum, linedata, text=None):
        self.enum = enum
        self.linedata = linedata
        self.footnotes = []
        if text:
            content = [TextNode(text)]
        else:
            content = []
        self.extend(content)
        self.logger = logbook.Logger(level='DEBUG')

    def __repr__(self):
        return 'Node(%r, %s)' % (self.enum, list.__repr__(self))

    @CachedAttribute
    def _new_child(self):
        '''Return a subclass of this class where the attribute `parent`
        points back to this class, enabling `append` to recursively
        move back up the tree structure if necessary.
        '''
        self_cls = self.__class__
        attrs = dict(parent=self)
        cls = type(self_cls.__name__, (self_cls,), attrs)
        return cls

    def _force_append(self, token, listappend=list.append):
        '''Append a child node without attempting to judge whether it
        fits or propagating it back up the tree.'''
        enum, text, linedata = token
        new_node = self._new_child(enum, linedata, text)

        # Append the new node to the parent.
        listappend(self, new_node)

        # Make the new node accessible via the token inside the parser.
        token.node = new_node
        return new_node

    def append(self, token,
               h=Enum('h'), H=Enum('H'),
               i=Enum('i'), I=Enum('I'),
               j=Enum('j'), J=Enum('J')):

        enum, text, linedata = token
        self_enum = self.enum

        try:
            is_root = self.is_root
        except AttributeError:
            pass
        else:
            if is_root:
                if enum:
                    return self._force_append(token)

        if enum is None:
            return self._force_append(token)

        if enum.is_first_in_scheme():

            # Disambiguate roman and alpahabetic 'i' and 'I'
            if enum == i and self_enum == h:
                # If the next enum is 'ii', the scheme is probably
                # 'lower_roman'.
                next_ = self.parser.stream.ahead(1)
                if next_.enum.could_be_next_after(enum):
                    return self.parent._force_append(token)

                # Else if it's 'j', it's probably just 'lower'.
                if next_.enum == j:
                    return self.parent._force_append(token)

            elif enum == I and self_enum == H:

                # If the next enum is 'II', the scheme is probably
                # 'upper_roman'.
                next_ = self.parser.stream.ahead(1)
                if next_.enum.could_be_next_after(enum):
                    return self.parent._force_append(token)

                # Else if it's 'J', it's probably just 'upper'.
                if next_.enum == J:
                    return self.parent._force_append(token)

            return self._force_append(token)

        if enum.was_nested:
            return self._force_append(token)

        if self_enum is None:
            self.parent._force_append(token)

        elif enum.could_be_next_after(self_enum):
            return self.parent._force_append(token)

        # If we get here, the previous append attempts all failed,
        # so propagate this node up to the current node's parent
        # and start over.
        return self.parent.append(token)

    def tree(self, indent=0):
        if self.linedata:
            print ' ' * indent, self.enum  # '[{0}, {1}]'.format(*self.linedata)
            # if 16 < int(self.linedata.arg):
            #     import pdb;pdb.set_trace()
        else:
            print ' ' * indent, self.enum
        if self.footnotes:
            for note in self.footnotes:
                print 'NOTE:', note['number'], note['offset'],repr(note['text'])
        for node in self:
            if isinstance(node, Node):
                node.tree(indent=indent + 2)
            elif isinstance(node, TextNode):
                if node.content is not None:
                    print ' ' * indent, node.content.encode('utf-8')

    def filedump(self, fp, indent=0):

        fp.write(' ' * indent)
        if self.enum:
            fp.write('(%s)' % self.enum.text)  # '[{0}, {1}]'.format(*self.linedata)
        for node in self:
            if isinstance(node, Node):
                node.filedump(fp, indent=indent + 2)
            elif isinstance(node, TextNode):
                if node.content is not None:
                    fp.write(' ')
                    fp.write(node.content.encode('utf-8'))

    def json(self):
        return dict(type='node', sub=[node.json() for node in self])

class Parser(object):

    SKIP = 1

    def __init__(self, stream):
        stream = Stream(stream)
        node_cls = type('Node', (Node,), dict(stream=stream, parser=self))
        root = node_cls(None, None, None)
        root.is_root = True
        self.root = root
        self.stream = stream

    def parse(self):

        this = self.root
        before_append = self.before_append
        after_append = self.after_append
        SKIPPED = 1
        for token in self.stream:

            # Try and execute beford_append.
            appended_to = before_append(token)
            if appended_to is not None:
                pass
            else:
                appended_to = this.append(token)

            after_append(token)

            # Change the parser's state.
            if appended_to is not SKIPPED and appended_to.enum:
                this = appended_to

        return self.root

    def before_append(self, token):
        raise NotImplemented

    def after_append(self, token):
        raise NotImplemented

class GPOLocatorParser(Parser):

    def __init__(self, *args, **kwargs):

        super(GPOLocatorParser, self).__init__(*args, **kwargs)

        # The parser needs to remember the last token it saw for
        # each gpo locator code encountered.
        self.codemap = {}

        # Mapping of encountered footnote numbers to nodes in which
        # they appear.
        self.footnotes = {}

    def before_append(self, token, finditer=re.finditer):

        SKIPPED = 1
        codemap = self.codemap

        # Keep track of the most recent token for each codearg.
        linedata = token.linedata
        if linedata is not None:
            # Nested enums have no linedata.
            codemap[linedata.codearg] = token

        # I13 tail text is denoted with I32.
        linedata = token.linedata
        if linedata is not None:

            codearg = token.linedata.codearg
            if codearg == 'I32':

                # Get the most recent I13 node.
                node = self.codemap['I13'].node
                return node.parent._force_append(token)

            if codearg == 'I17':
                node = self.codemap['I12'].node
                return node.parent._force_append(token)

            if codearg == 'I28':
                return SKIPPED

        # Associate nodes with footnotes they contain.
        if token.text:
            for matchobj in finditer(r'\\(\d+)\\\x07N', token.text):
                number, offset = self.matchobj_to_notedata(token, matchobj)
                self.footnotes[number] = (token, offset)

    def after_append(self, token):

        # Match footnotes to their annotations in node text.
        linedata = token.linedata
        if linedata and linedata.codearg == 'I28':
            target_token, note = self.token_to_note(token)
            target_token.node.footnotes.append(note)

    #  methods.
    def matchobj_to_notedata(self, token, matchobj):
        offset = matchobj.start()
        number = matchobj.group(1)
        return number, offset

    def token_to_note(self, token):
        text = token.text

        # If the text doesn't start with\x07, it's not a note.
        assert text[0] == '\x07'

        number = text[3]
        target_token, offset = self.footnotes[number]
        text = re.sub(r'^\x07N\\(\d+)\\\s+', '', text)
        note = dict(offset=offset, text=text, number=number)
        return target_token, note


text:
from operator import itemgetter

## {{{ http://code.activestate.com/recipes/276643/ (r1)
class CachedAttribute(object):
    '''Computes attribute value and caches it in instance.

    Example:
        class MyClass(object):
            def myMethod(self):
                # ...
            myMethod = CachedAttribute(myMethod)
    Use "del inst.myMethod" to clear cache.'''

    def __init__(self, method, name=None):
        self.method = method
        self.name = name or method.__name__

    def __get__(self, inst, cls):
        if inst is None:
            return self
        result = self.method(inst)
        setattr(inst, self.name, result)
        return result

class NiceList(list):

    first = property(itemgetter(0))
    second = property(itemgetter(1))
    third = property(itemgetter(2))
    fourth = property(itemgetter(3))
    fifth = property(itemgetter(4))
    sixth = property(itemgetter(5))

    rest = property(itemgetter(slice(1, None)))

repo_name: unitedstates/uscode

repo_language: Python


text:
project(queercat)
add_executable(queercat main.c)
target_link_libraries(queercat
    m)

repo_language: C

mime_type: text/plain


text:
#define _XOPEN_SOURCE
#define _GNU_SOURCE

/* *** Includes ******************************************************/
#include <stdbool.h>
#include <ctype.h>
#include <err.h>
#include <errno.h>
#include <locale.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/time.h>
#include <unistd.h>
#include <wchar.h>
#include <time.h>
#include "math.h"

/* *** Common ********************************************************/
/* Constants */
#define NEWLINE '\n'
#define ESCAPE_CHAR '\033'

/* Types */
typedef enum escape_state_e {
    ESCAPE_STATE_OUT = 0,
    ESCAPE_STATE_IN,
    ESCAPE_STATE_LAST
} escape_state_t;

/* Macros */
#define UNUSED(var) ((void)(var))
#define NEXT_CYCLIC_ELEMENT(array, index, array_size) \
    (((index) + 1 == (array_size)) ? (array)[0] : (array)[((index) + 1)] )
#define IS_LETTER(c) (('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z'))

/* *** Constants *****************************************************/

#define MAX_FLAG_STRIPES (6)
#define MAX_ANSII_CODES_PER_STRIPE (5)
#define MAX_ANSII_CODES_COUNT (MAX_FLAG_STRIPES * MAX_ANSII_CODES_PER_STRIPE)
#define MAX_FLAG_NAME_LENGTH (64)

/* *** Types *********************************************************/
/* Colors. */
typedef uint32_t hex_color_t;
typedef unsigned char ansii_code_t;
typedef struct color_s {
    uint8_t red;
    uint8_t green;
    uint8_t blue;
} color_t;

/* Color type patterns. */
typedef enum color_type_e {
    COLOR_TYPE_INVALID = -1,
    COLOR_TYPE_ANSII = 0,
    COLOR_TYPE_24_BIT,
    COLOR_TYPE_COUNT
} color_type_t;
typedef struct ansii_pattern_s {
    const unsigned int codes_count;
    const unsigned char ansii_codes[MAX_ANSII_CODES_COUNT];
} ansii_pattern_t;
typedef struct color_pattern_s {
    const uint8_t stripes_count;
    const uint32_t stripes_colors[MAX_FLAG_STRIPES];
    const float factor;
} color_pattern_t;

/* Get color function. */
typedef void(get_color_f)(const color_pattern_t *color_pattern, float theta, color_t *color);

/* Pattern. */
typedef struct pattern_s {
    const char name[MAX_FLAG_NAME_LENGTH];
    const ansii_pattern_t ansii_pattern;
    const color_pattern_t color_pattern;
    get_color_f *get_color;
} pattern_t;

/* *** A Single Global ***********************************************/
char *helpstr;

/* *** Pattern Functions *********************************************/
get_color_f get_color_rainbow;
get_color_f get_color_stripes;

/* *** Flags *********************************************************/
const pattern_t flags[] = {
    {
        .name = "rainbow",
        .ansii_pattern = {
            .codes_count = 30,
            .ansii_codes = { 39, 38, 44, 43, 49, 48, 84, 83, 119, 118, 154, 148, 184, 178,
                214, 208, 209, 203, 204, 198, 199, 163, 164, 128, 129, 93, 99, 63, 69, 33 }
        },
        .color_pattern = { 0 },
        .get_color = get_color_rainbow
    },

    {
        .name = "transgender",
        .ansii_pattern = {
            .codes_count = 10,
            .ansii_codes = {81, 81, 217, 217,  231, 231,  217, 217,  81, 81}
        },
        .color_pattern = {
            .stripes_count = 5,
            .stripes_colors = {
                0x55cdfc, /* #55cdfc - Blue */
                0xf7a8b8, /* #f7a8b8 - Pink */
                0xffffff, /* #ffffff - White */
                0xf7a8b8, /* #f7a8b8 - Pink */
                0x55cdfc  /* #55cdfc - Blue */
            },
            .factor = 4.0f
        },
        .get_color = get_color_stripes
    },

    {
        .name = "nonbinary",
        .ansii_pattern = {
            .codes_count = 8,
            .ansii_codes = {226, 226, 255, 255, 93, 93, 234, 234}
        },
        .color_pattern = {
            .stripes_count = 4,
            .stripes_colors = {
                0xffff00, /* #ffff00 - Yellow */
                0xb000ff, /* #b000ff - Purple */
                0xffffff, /* #ffffff - White */
                0x000000  /* #000000 - Black */
            },
            .factor = 4.0f
        },
        .get_color = get_color_stripes
    },

    {
        .name = "lesbian",
        .ansii_pattern = {
            .codes_count = 5,
            .ansii_codes = {196, 208, 255, 170, 128}
        },
        .color_pattern = {
            .stripes_count = 5,
            .stripes_colors = {
                0xff0000, /* #ff0000 - Red */
                0xff993f, /* #ff993f - Orange */
                0xffffff, /* #ffffff - White */
                0xff8cbd, /* #ff8cbd - Pink */
                0xff4284  /* #ff4284 - Purple */
            },
            .factor = 2.0f
        },
        .get_color = get_color_stripes
    },

    {
        .name = "gay",
        .ansii_pattern = {
            .codes_count = 7,
            .ansii_codes = {36, 49, 121, 255, 117, 105, 92}
        },
        .color_pattern = {
            .stripes_count = 5,
            .stripes_colors = {
                0x00b685, /* #00b685 - Teal */
                0x6bffb6, /* #6bffb6 - Green */
                0xffffff, /* #ffffff - White */
                0x8be1ff, /* #8be1ff - Blue */
                0x8e1ae1  /* #8e1ae1 - Purple */
            },
            .factor = 6.0f
        },
        .get_color = get_color_stripes
    },

    {
        .name = "pansexual",
        .ansii_pattern = {
            .codes_count = 9,
            .ansii_codes = {200, 200, 200,  227, 227, 227,  45, 45, 45}
        },
        .color_pattern = {
            .stripes_count = 3,
            .stripes_colors = {
                0xff3388, /* #ff3388 - Pink */
                0xffea00, /* #ffea00 - Yellow */
                0x00dbff  /* #00dbff - Cyan */
            },
            .factor = 8.0f
        },
        .get_color = get_color_stripes
    },

    {
        .name = "bisexual",
        .ansii_pattern = {
            .codes_count = 8,
            .ansii_codes = {162, 162, 162,  129, 129, 27, 27, 27}
        },
        .color_pattern = {
            .stripes_count = 5,
            .stripes_colors = {
                0xff3b7b, /* #ff3b7b - Pink */
                0xff3b7b, /* #ff3b7b - Pink */
                0xd06bcc, /* #d06bcc - Purple */
                0x3b72ff, /* #3b72ff - Blue */
                0x3b72ff  /* #3b72ff - Blue */
            },
            .factor = 4.0f
        },
        .get_color = get_color_stripes
    },

    {
        .name = "gender_fluid",
        .ansii_pattern = {
            .codes_count = 10,
            .ansii_codes = {219, 219, 255, 255, 128, 128, 234, 234, 20, 20}
        },
        .color_pattern = {
            .stripes_count = 5,
            .stripes_colors = {
                0xffa0bc, /* #ffa0bc - Pink */
                0xffffff, /* #ffffff - White */
                0xc600e4, /* #c600e4 - Purple */
                0x000000, /* #000000 - Black */
                0x4e3cbb  /* #4e3cbb - Blue */
            },
            .factor = 2.0f
        },
        .get_color = get_color_stripes
    },

    {
        .name = "asexual",
        .ansii_pattern = {
            .codes_count = 8,
            .ansii_codes = {233, 233, 247, 247, 255, 255, 5, 5}
        },
        .color_pattern = {
            .stripes_count = 4,
            .stripes_colors = {
                0x000000, /* #000000 - Black */
                0xa3a3a3, /* #a3a3a3 - Gray */
                0xffffff, /* #ffffff - White */
                0x800080  /* #800080 - Purple */
            },
            .factor = 4.0f
        },
        .get_color = get_color_stripes
    },

    {
        .name = "unlabeled",
        .ansii_pattern = {
            .codes_count = 8,
            .ansii_codes = {194, 194, 255, 255, 195, 195, 223, 223}
        },
        .color_pattern = {
            .stripes_count = 4,
            .stripes_colors = {
                0xe6f9e3, /* #e6f9e3 - Green */
                0xfdfdfb, /* #fdfdfb - White */
                0xdeeff9, /* #deeff9 - Blue */
                0xfae1c2  /* #fae1c2 - Orange */
            },
            .factor = 4.0f
        },
        .get_color = get_color_stripes
    },

    {
        .name = "aromantic",
        .ansii_pattern = {
            .codes_count = 10,
            .ansii_codes = {
                34, 34,
                120, 120,
                255, 255,
                247, 247,
                233, 233
            }
        },
        .color_pattern = {
            .stripes_count = 5,
            .stripes_colors = {
                0x3da542, /* #3da542 - Green        */
                0xa8d379, /* #a8d379 - Light green  */
                0xffffff, /* #ffffff - White        */
                0xa9a9a9, /* #a9a9a9 - Grey         */
                0x000000  /* #000000 - Black        */
            },
            .factor = 1.0f
        },
        .get_color = get_color_stripes
    },

    {
        .name = "aroace",
        .ansii_pattern = {
            .codes_count = 10,
            .ansii_codes = {
                208, 208,
                220, 220,
                255, 255,
                75, 75,
                62, 62
            },
        },
        .color_pattern = {
            .stripes_count = 5,
            .stripes_colors = {
                0xe28d00, /* #e28d00 - Orange     */
                0xeccd00, /* #eccd00 - Yellow     */
                0xffffff, /* #ffffff - White      */
                0x62afdd, /* #62afdd - Light blue */
                0x203756  /* #203756 - Blue       */
            },
            .factor = 1.0f
        },
        .get_color = get_color_stripes
    },
    /* Add new flags above this line. */
};

const int FLAG_COUNT = sizeof(flags)/sizeof(flags[0]);

/* *** Functions Declarations ****************************************/
/* Info */
static void usage(void);
static void version(void);

/* Helpers */
static void build_helpstr(void);
static void cleanup_helpstr(void);
static void find_escape_sequences(wint_t current_char, escape_state_t *state);
static wint_t helpstr_hack(FILE * _ignored);
static const pattern_t * lookup_pattern(const char *name);

/* Colors handling */
static void mix_colors(uint32_t color1, uint32_t color2, float balance, float factor, color_t *output_color);
static void print_color(const pattern_t *pattern, color_type_t color_type, int char_index, int line_index, double freq_h, double freq_v, double offx, int rand_offset, int cc);

/* *** Functions *****************************************************/
static void usage(void)
{
    wprintf(L"Usage: queercat [-h horizontal_speed] [-v vertical_speed] [--] [FILES...]\n");
    exit(1);
}

static void version(void)
{
    wprintf(L"queercat version 2.0, (c) 2022 elsa002\n");
    exit(0);
}

static void build_helpstr(void)
{
    if(helpstr != NULL)
        return;

    static char helpstr_head[] = "\n"
        "Usage: queercat [-f flag_number][-h horizontal_speed] [-v vertical_speed] [--] [FILES...]\n"
        "\n"
        "Concatenate FILE(s), or standard input, to standard output.\n"
        "With no FILE, or when FILE is -, read standard input.\n"
        "\n"
        "                --flag <d>, -f <d>: Choose colors to use (default: 0 (rainbow)):\n";

    static char helpstr_indent[] = "                                      ";

    static char helpstr_tail[] =
        "--horizontal-frequency <d>, -h <d>: Horizontal rainbow frequency (default: 0.23)\n"
        "  --vertical-frequency <d>, -v <d>: Vertical rainbow frequency (default: 0.1)\n"
        "              --offset <d>, -o <d>: Offset of the start of the flag\n"
        "                 --force-color, -F: Force color even when stdout is not a tty\n"
        "             --no-force-locale, -l: Use encoding from system locale instead of\n"
        "                                    assuming UTF-8\n"
        "                      --random, -r: Random colors\n"
        "                       --24bit, -b: Output in 24-bit \"true\" RGB mode (slower and\n"
        "                                    not supported by all terminals)\n"
        "                         --version: Print version and exit\n"
        "                            --help: Show this message\n"
        "\n"
        "Examples:\n"
        "  queercat f - g      Output f's contents, then stdin, then g's contents.\n"
        "  queercat            Copy standard input to standard output.\n"
        "  fortune | queercat  Display a rainbow cookie.\n"
        "\n"
        "Report queercat bugs to <https://github.com/elsa002/queercat/issues>\n"
        "queercat home page: <https://github.com/elsa002/queercat/>\n"
        "base for code: <https://github.com/jaseg/lolcat/>\n"
        "Original idea: <https://github.com/busyloop/lolcat/>\n";

    /* old version of what this generates, for reference:
     * "                                    [rainbow: 0, trans: 1, NB: 2, lesbian: 3,\n"
     * "                                    gay: 4, pan: 5, bi: 6, genderfluid: 7, asexual: 8,\n"
     * "                                    unlabeled: 9, aromantic: 10, aroace: 11]\n"
     * would be nice to have the dynamic word-wrap back, but that's
     * more clever than I currently feel like trying to be
     */
    const int line_max_len = strlen(helpstr_indent) + MAX_FLAG_NAME_LENGTH + strlen(": 000\n") ;
    char lines[FLAG_COUNT][line_max_len];
    size_t lines_total_len = 0;

    for(int i = 0; i < FLAG_COUNT; ++i) {
        lines_total_len += snprintf(lines[i], line_max_len, "%s%s: %d\n", helpstr_indent, flags[i].name, i);
    }

    size_t helpstr_len = strlen(helpstr_head) + lines_total_len + strlen(helpstr_tail);

    char *out = malloc(helpstr_len);
    char *out_pos = out;

    out_pos = mempcpy(out, helpstr_head, strlen(helpstr_head));

    for(int i = 0; i < FLAG_COUNT; ++i) {
        char* this_line = lines[i];
        out_pos = mempcpy(out_pos, this_line, strlen(this_line));
    }

    memcpy(out_pos, helpstr_tail, strlen(helpstr_tail));

    // TODO? maybe refactor helpstr_hack to not rely on helpstr being global to avoid needing to use atexit
    helpstr = out;
    atexit(cleanup_helpstr);
}

static void cleanup_helpstr(void)
{
    free(helpstr);
}

static void find_escape_sequences(wint_t current_char, escape_state_t *state)
{
    if (current_char == '\033') {
        *state = ESCAPE_STATE_IN;
    } else if (*state == ESCAPE_STATE_IN) {
        *state = IS_LETTER(current_char) ? ESCAPE_STATE_LAST : ESCAPE_STATE_IN;
    } else {
        *state = ESCAPE_STATE_OUT;
    }
}

static wint_t helpstr_hack(FILE * _ignored)
{
    (void)_ignored;
    static size_t idx = 0;
    char c = helpstr[idx++];
    if (c)
        return c;
    idx = 0;
    return WEOF;
}

/* returns NULL on failure */
static const pattern_t * lookup_pattern(const char *name)
{
    // check for name matches
    for(int i = 0; i < FLAG_COUNT; ++i) {
        if(!strcmp(name, flags[i].name))
            return &flags[i];
    }

    // try number matches
    char *endptr;
    int flag_num = (int)strtoul(name, &endptr, 10);

    // left-over charaters, or number out of range
    if(*endptr || (flag_num < 0 || flag_num >= FLAG_COUNT))
        return NULL;

    return &flags[flag_num];
}

static void mix_colors(uint32_t color1, uint32_t color2, float balance, float factor, color_t *output_color)
{
    uint8_t red_1   = (color1 & 0xff0000) >> 16;
    uint8_t green_1 = (color1 & 0x00ff00) >>  8;
    uint8_t blue_1  = (color1 & 0x0000ff) >>  0;

    uint8_t red_2   = (color2 & 0xff0000) >> 16;
    uint8_t green_2 = (color2 & 0x00ff00) >>  8;
    uint8_t blue_2  = (color2 & 0x0000ff) >>  0;

    balance = pow(balance, factor);

    output_color->red = lrintf(red_1 * balance + red_2 * (1.0f - balance));
    output_color->green = lrintf(green_1 * balance + green_2 * (1.0f - balance));
    output_color->blue = lrintf(blue_1 * balance + blue_2 * (1.0f - balance));
}

void get_color_rainbow (const color_pattern_t *color_pattern, float theta, color_t *color)
{
    /* Unused variables. */
    UNUSED(color_pattern);

    /* Get theta in range. */
    while (theta < 0) theta += 2.0f * (float)M_PI;
    while (theta >= 2.0f * (float)M_PI) theta -= 2.0f * (float)M_PI;

    /* Generate the color. */
    color->red   = lrintf((1.0f * (0.5f + 0.5f * sin(theta + 0            ))) * 255.0f);
    color->green = lrintf((1.0f * (0.5f + 0.5f * sin(theta + 2 * M_PI / 3 ))) * 255.0f);
    color->blue  = lrintf((1.0f * (0.5f + 0.5f * sin(theta + 4 * M_PI / 3 ))) * 255.0f);
}

void get_color_stripes (const color_pattern_t *color_pattern, float theta, color_t *color)
{
    /* Get theta in range. */
    while (theta < 0) theta += 2.0f * (float)M_PI;
    while (theta >= 2.0f * (float)M_PI) theta -= 2.0f * (float)M_PI;

    /* Find the stripe based on theta and generate the color. */
    for (int i = 0; i < color_pattern->stripes_count; i++) {
        float stripe_size = (2.0f * M_PI) / color_pattern->stripes_count;
        float min_theta = i * stripe_size;
        float max_theta = (i + 1) * stripe_size;

        if (min_theta <= theta && max_theta > theta) {
            float balance = 1 - ((theta - min_theta) / stripe_size);
            mix_colors(
                    color_pattern->stripes_colors[i],
                    NEXT_CYCLIC_ELEMENT(color_pattern->stripes_colors, i, color_pattern->stripes_count),
                    balance,
                    color_pattern->factor,
                    color);
            return;
        }
    }
}

static void print_color(const pattern_t *pattern, color_type_t color_type, int char_index, int line_index, double freq_h, double freq_v, double offx, int rand_offset, int cc)
{
    float theta;
    color_t color = { 0 };

    int ncc;

    switch (color_type) {
        case COLOR_TYPE_24_BIT:
            theta = char_index * freq_h / 5.0f + line_index * freq_v + (offx + 2.0f * rand_offset / (float)RAND_MAX) * M_PI;

            pattern->get_color(&pattern->color_pattern, theta, &color);
            wprintf(L"\033[38;2;%d;%d;%dm", color.red, color.green, color.blue);
            break;

        case COLOR_TYPE_ANSII:
            ncc = offx * pattern->ansii_pattern.codes_count + (int)(char_index * freq_h + line_index * freq_v);
            if (cc != ncc)
                wprintf(L"\033[38;5;%hhum", pattern->ansii_pattern.ansii_codes[(rand_offset + (cc = ncc)) % pattern->ansii_pattern.codes_count]);
            break;

        default:
            exit(1);
    }
}

int main(int argc, char** argv)
{
    char* default_argv[] = { "-" };
    int cc = -1;
    int i = 0;
    int char_index = 0;
    int line_index = 0;
    wint_t current_char = '\0';
    bool print_colors = isatty(STDOUT_FILENO);
    bool force_locale = true;
    bool random = false;
    color_type_t color_type = COLOR_TYPE_ANSII;
    double freq_h = 0.23;
    double freq_v = 0.1;
    char* flag_type = "rainbow";

    struct timeval tv;
    gettimeofday(&tv, NULL);
    double offx = (tv.tv_sec % 300) / 300.0;

    build_helpstr();

    /* Handle flags. */
    for (i = 1; i < argc; i++) {
        char* endptr;
        if (!strcmp(argv[i], "-f") || !strcmp(argv[i], "--flag")) {
            if ((++i) < argc) {
                flag_type = argv[i];
            } else {
                usage();
            }
        } else if (!strcmp(argv[i], "-h") || !strcmp(argv[i], "--horizontal-frequency")) {
            if ((++i) < argc) {
                freq_h = strtod(argv[i], &endptr);
                if (*endptr)
                    usage();
            } else {
                usage();
            }
        } else if (!strcmp(argv[i], "-o") || !strcmp(argv[i], "--offset")) {
            if ((++i) < argc) {
                offx = strtod(argv[i], &endptr);
                if (*endptr)
                    usage();
            } else {
                usage();
            }
        } else if (!strcmp(argv[i], "-v") || !strcmp(argv[i], "--vertical-frequency")) {
            if ((++i) < argc) {
                freq_v = strtod(argv[i], &endptr);
                if (*endptr)
                    usage();
            } else {
                usage();
            }
        } else if (!strcmp(argv[i], "-F") || !strcmp(argv[i], "--force-color")) {
            print_colors = true;
        } else if (!strcmp(argv[i], "-l") || !strcmp(argv[i], "--no-force-locale")) {
            force_locale = false;
        } else if (!strcmp(argv[i], "-r") || !strcmp(argv[i], "--random")) {
            random = true;
        } else if (!strcmp(argv[i], "-b") || !strcmp(argv[i], "--24bit")) {
            color_type = COLOR_TYPE_24_BIT;
        } else if (!strcmp(argv[i], "--version")) {
            version();
        } else {
            if (!strcmp(argv[i], "--"))
                i++;
            break;
        }
    }

    /* Get pattern. */
    const pattern_t *pattern = lookup_pattern(flag_type);
    if (pattern == NULL) {
        fprintf(stderr, "Invalid flag: %s\n", flag_type);
        exit(1);
    }

    /* Handle randomness. */
    int rand_offset = 0;
    if (random) {
        srand(time(NULL));
        rand_offset = rand();
    }

    /* Get inputs. */
    char** inputs = argv + i;
    char** inputs_end = argv + argc;
    if (inputs == inputs_end) {
        inputs = default_argv;
        inputs_end = inputs + 1;
    }

    /* Handle locale. */
    char* env_lang = getenv("LANG");
    if (force_locale && env_lang && !strstr(env_lang, "UTF-8")) {
        if (!setlocale(LC_ALL, "C.UTF-8")) { /* C.UTF-8 may not be available on all platforms */
            setlocale(LC_ALL, ""); /* Let's hope for the best */
        }
    } else {
        setlocale(LC_ALL, "");
    }

    /* For file in inputs. */
    for (char** filename = inputs; filename < inputs_end; filename++) {
        wint_t (*this_file_read_wchar)(FILE*); /* Used for --help because fmemopen is universally broken when used with fgetwc */
        FILE* f;
        escape_state_t escape_state = ESCAPE_STATE_OUT;

        /* Handle "--help", "-" (STDIN) and file names. */
        if (!strcmp(*filename, "--help")) {
            this_file_read_wchar = &helpstr_hack;
            f = 0;

        } else if (!strcmp(*filename, "-")) {
            this_file_read_wchar = &fgetwc;
            f = stdin;

        } else {
            this_file_read_wchar = &fgetwc;
            f = fopen(*filename, "r");
            if (!f) {
                fwprintf(stderr, L"Cannot open input file \"%s\": %s\n", *filename, strerror(errno));
                return 2;
            }
        }

        /* While there are chars to read. */
        while ((current_char = this_file_read_wchar(f)) != WEOF) {

            /* If set to print colors, handle the colors. */
            if (print_colors) {

                /* Skip escape sequences. */
                find_escape_sequences(current_char, &escape_state);
                if (escape_state == ESCAPE_STATE_OUT) {

                    /* Handle newlines. */
                    if (current_char == '\n') {
                        line_index++;
                        char_index = 0;
                    } else {
                        char_index += wcwidth(current_char);
                        print_color(pattern, color_type, char_index, line_index, freq_h, freq_v, offx, rand_offset, cc);
                    }
                }
            }

            /* Print the char. */
            putwchar(current_char);

            if (escape_state == ESCAPE_STATE_LAST) {  /* implies "print_colors" */
                print_color(pattern, color_type, char_index, line_index, freq_h, freq_v, offx, rand_offset, cc);
            }
        }

        if (print_colors)
            wprintf(L"\033[0m");

        cc = -1;

        if (f) {
            if (ferror(f)) {
                fwprintf(stderr, L"Error reading input file \"%s\": %s\n", *filename, strerror(errno));
                fclose(f);
                return 2;
            }

            if (fclose(f)) {
                fwprintf(stderr, L"Error closing input file \"%s\": %s\n", *filename, strerror(errno));
                return 2;
            }
        }
    }
}


text:
# queercat
a version of lolcat with some lgbtq+ pride flags options

## Usage
`$ queercat [-f flag_number][-h horizontal_speed] [-v vertical_speed] [--] [FILES...]`

```
Concatenate FILE(s), or standard input, to standard output.
With no FILE, or when FILE is -, read standard input.

--flag <d>                , -f <d>: Choose colors to use: [rainbow: 0, trans: 1, NB: 2, lesbian: 3, gay: 4, pan: 5, bi: 6, genderfluid: 7, asexual: 8, unlabeled: 9, aromantic : 10, aroace: 11] default is rainbow(0)
--horizontal-frequency <d>, -h <d>: Horizontal rainbow frequency (default: 0.23)
  --vertical-frequency <d>, -v <d>: Vertical rainbow frequency (default: 0.1)
                 --force-color, -F: Force color even when stdout is not a tty
             --no-force-locale, -l: Use encoding from system locale instead of assuming UTF-8
                    --random, -r: Random colors
                       --24bit, -b: Output in 24-bit "true" RGB mode (slower and
                                    not supported by all terminals)
                         --version: Print version and exit
                            --help: Show this message
```
(Flag list above may not be up to date; run `queercat --help` to see which flags your version supports!)

## Adding a flag
### Step 1: Define the pattern
To add a flag, first add an instance of `pattern_t` for it to the `flags` array in the `main.c` file.
Look for the section `/* *** Flags *********************************************************/`, then find
`/* Add new flags above this line. */`. The order is important! For the sake of simplicity, you should
only add to the end.

Example:
``` c
    },

    {
        .name = "nonbinary",
            .ansii_pattern = {
                .codes_count = 8,
                .ansii_codes = {226, 226, 255, 255, 93, 93, 234, 234}
            },
            .color_pattern = {
                .stripes_count = 4,
                .stripes_colors = {
                    0xffff00, /* #ffff00 - Yellow */
                    0xb000ff, /* #b000ff - Purple */
                    0xffffff, /* #ffffff - White */
                    0x000000  /* #000000 - Black */
                },
                .factor = 4.0f
            },
            .get_color = get_color_stripes
    },
    /* Add new flags above this line. */
};
```

### Step 2 (optional): Update `README.md` to include your addition.
**Pay attention to the number it gets from its position in the array!**

Extend the line for `--flag` under `Usage`.

*Note that in the readme it is a single line.*

### Step 3: Pull request :)

## Compiling
to compile with gcc: `$ gcc main.c -lm -o queercat`

add the binary to a directory in your `PATH` variable (`/bin` can work) to use from everywhere

## Credits
base for code: <https://github.com/jaseg/lolcat/>
Original idea: <https://github.com/busyloop/lolcat/>


text:
/// <reference types="next" />
/// <reference types="next/image-types/global" />

// NOTE: This file should not be edited
// see https://nextjs.org/docs/basic-features/typescript for more information.

repo_name: Jordan-Gilliam/ai-template

repo_language: TypeScript


text:
/** @type {import('prettier').Config} */
/** @type {import("@ianvs/prettier-plugin-sort-imports").PrettierConfig} */

module.exports = {
  endOfLine: "lf",
  semi: false,
  singleQuote: false,
  tabWidth: 2,
  trailingComma: "es5",
  importOrder: [
    "^(react/(.*)$)|^(react$)",
    "^(next/(.*)$)|^(next$)",
    "<THIRD_PARTY_MODULES>",

    "^types$",
    "^@/types/(.*)$",
    "^@/config/(.*)$",
    "^@/lib/(.*)$",

    "^@/components/(.*)$",
    "^@/hooks/(.*)$",
    "^@/styles/(.*)$",
  ],
  importOrderSeparation: false,
  importOrderSortSpecifiers: true,
  importOrderBuiltinModulesToTop: true,
  importOrderParserPlugins: ["typescript", "jsx", "decorators-legacy"],
  importOrderMergeDuplicateImports: true,
  importOrderCombineTypeAndValueImports: true,
  plugins: [
    "@ianvs/prettier-plugin-sort-imports",
    require("prettier-plugin-tailwindcss"),
  ],
}


text:
const { fontFamily } = require("tailwindcss/defaultTheme")

/** @type {import('tailwindcss').Config} */
module.exports = {
  darkMode: ["class"],
  important: true,
  content: ["pages/**/*.{ts,tsx}", "components/**/*.{ts,tsx}"],
  theme: {
    container: {
      center: true,
      padding: "1.5rem",
      screens: {
        "2xl": "1360px",
      },
    },
    extend: {
      typography: {
        DEFAULT: {
          css: {
            "code::before": {
              content: '""',
            },
            "code::after": {
              content: '""',
            },
          },
        },
      },
      boxShadow: {
        input:
          "0px 1px 0px -1px var(--tw-shadow-color), 0px 1px 1px -1px var(--tw-shadow-color), 0px 1px 2px -1px var(--tw-shadow-color), 0px 2px 4px -2px var(--tw-shadow-color), 0px 3px 6px -3px var(--tw-shadow-color)",
        highlight:
          "inset 0px 0px 0px 1px var(--tw-shadow-color), inset 0px 1px 0px var(--tw-shadow-color)",
      },
      fontFamily: {
        sans: ["var(--font-sans)", ...fontFamily.sans],
        aboreto: ["var(--font-aboreto)", ...fontFamily.sans],
      },
      keyframes: {
        "accordion-down": {
          from: { height: 0 },
          to: { height: "var(--radix-accordion-content-height)" },
        },
        "accordion-up": {
          from: { height: "var(--radix-accordion-content-height)" },
          to: { height: 0 },
        },
      },
      colors: {
        neutral: {
          750: "#313131",
        },
      },

      animation: {
        "accordion-down": "accordion-down 0.2s ease-out",
        "accordion-up": "accordion-up 0.2s ease-out",
      },
    },
  },
  plugins: [
    require("windy-radix-palette"),
    require("tailwindcss-animate"),
    require("@tailwindcss/typography"),
    require("@tailwindcss/forms"),
  ],
  presets: [require("windy-radix-palette")],
}


text:
/** @type {import('next').NextConfig} */
const nextConfig = {
  reactStrictMode: true,
  // experimental: {
  //   fontLoaders: [
  //     {
  //       loader: "@next/font/google",
  //     },
  //   ],
  // },
}

export default nextConfig


text:
module.exports = {
  plugins: {
    tailwindcss: {},
    autoprefixer: {},
  },
}


text:
Mercury

## Chat with any Document or Website
> Train your own custom GPT

- Train on specific websites that you define
- Train on documents you upload
- Builds on dialog with Chat History
- Cites sources
- [Perplexity](https://www.perplexity.ai/) style UI

#### Supported Files
- [x] .pdf
- [x] .docx
- [x] .md
- [x] .txt
- [x] .png
- [x] .jpg
- [x] .html
- [x] .json

#### Coming Soon
- [ ] .csv
- [ ] .pptx
- [ ] notion
- [ ] next 13 app dir
- [ ] vercel ai sdk

## Train

#### 1. Upload: `/api/embed-file`

- file is uploaded -> cleaned to plain text, and split into 1000-character documents.
- OpenAI's embedding API is used to generate embeddings for each document using the "text-embedding-ada-002" model.
- The embeddings are stored in a Pinecone namespace.

#### 2. Scrape: `/api/embed-webpage`

- Web pages are scraped using [cheerio](https://github.com/cheeriojs/cheerio), cleaned to plain text, and split into 1000-character documents.
- OpenAI's embedding API is used to generate embeddings for each document using the "text-embedding-ada-002" model.
- The embeddings are stored in a Pinecone namespace.

## Query

#### Responding to queries: `/api/query`

- A single embedding is generated from the user prompt.
- The embedding is used to perform a similarity search against the vector database.
- The results of the similarity search are used to construct a prompt for GPT-3.
- The GTP-3 response is then streamed back to the user.

## Getting Started

### 1. Clone Repo and Install Deps

To create a new project based on this template using [degit](https://github.com/Rich-Harris/degit):

```bash
npx degit https://github.com/Jordan-Gilliam/ai-template ai-template
```

```bash
cd ai-template
code .
```

- install dependencies

```bash
npm i
```

### 2. Set-up Pinecone

- Visit [pinecone](https://pinecone.io/) to create a free tier account and from the dashboard.
- Create a new Pinecone Index with Dimensions `1536`
eg:

- Copy your API key
- Record your Enironment name ex: `us-central1-gcp`
- Record your index name ex: `mercury`

### 3. Set-up OpenAi API

- Visit [openai](https://platform.openai.com/account/api-keys) to create and copy your API key
> You can find this in the OpenAI web portal under `API Keys`

### 4. Open the `.env.local` file and configure your environment

```bash
cp .env.example .env.local
```

```bash
# OpenAI
OPENAI_API_KEY="sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
# Pinecone
PINECONE_API_KEY="xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxx"
PINECONE_ENVIRONMENT="us-central1-gcp"
PINECONE_INDEX_NAME="mercury"
```

### 5. Start the app

```bash
npm run dev
```

Open http://localhost:3000 in your browser to view the app.

## Template Features

- OpenAI API (for generating embeddings and GPT-3 responses)
- Pinecone
- Nextjs API Routes (Edge runtime) - streaming
- Tailwind CSS
- Fonts with `@next/font`
- Icons from [Lucide](https://lucide.dev)
- Dark mode with `next-themes`
- Radix UI Primitives
- Automatic import sorting with `@ianvs/prettier-plugin-sort-imports`

## Inspiration:

> 🍴 Huge thanks to [@gannonh](https://github.com/gannonh) and [@mayooear](https://github.com/mayooear/gpt4-pdf-chatbot-langchain) for their fantastic work that helped inspire this template.

- https://www.perplexity.ai/
- https://builtbyjesse.com/
- https://ui.shadcn.com/docs
- https://meodai.github.io/poline/
- https://github.com/gannonh/gpt3.5-turbo-pgvector
- https://github.com/vercel/examples/tree/main/solutions/ai-chatgpt

## How embeddings work:

ChatGPT is a great tool for answering general questions, but it falls short when it comes to answering domain-specific questions as it often makes up answers to fill its knowledge gaps and doesn't cite sources. To solve this issue, this starter app uses embeddings coupled with vector search. This app shows how OpenAI's GPT-3 API can be used to create conversational interfaces for domain-specific knowledge.

Embeddings are vectors of floating-point numbers that represent the "relatedness" of text strings. They are very useful for tasks like ranking search results, clustering, and classification. In text embeddings, a high cosine similarity between two embedding vectors indicates that the corresponding text strings are highly related.

This app uses embeddings to generate a vector representation of a document and then uses vector search to find the most similar documents to the query. The results of the vector search are then used to construct a prompt for GPT-3, which generates a response. The response is then streamed back to the user.


text:
---
name: Bug report
about: Create a report to help us improve
title: ''
labels: ''
assignees: ''

---

**Describe the bug**
A clear and concise description of what the bug is.

**To Reproduce**
Steps to reproduce the behavior:
1. Go to '...'
2. Click on '....'
3. Scroll down to '....'
4. See error

**Expected behavior**
A clear and concise description of what you expected to happen.

**Screenshots**
If applicable, add screenshots to help explain your problem.

**Desktop (please complete the following information):**
 - OS: [e.g. iOS]
 - Browser [e.g. chrome, safari]
 - Version [e.g. 22]

**Smartphone (please complete the following information):**
 - Device: [e.g. iPhone6]
 - OS: [e.g. iOS8.1]
 - Browser [e.g. stock browser, safari]
 - Version [e.g. 22]

**Additional context**
Add any other context about the problem here.


text:
---
name: Feature request
about: Suggest an idea for this project
title: ''
labels: ''
assignees: ''

---

**Is your feature request related to a problem? Please describe.**
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]

**Describe the solution you'd like**
A clear and concise description of what you want to happen.

**Describe alternatives you've considered**
A clear and concise description of any alternative solutions or features you've considered.

**Additional context**
Add any other context or screenshots about the feature request here.


text:
import type { AppProps } from "next/app"
import { Aboreto, Work_Sans as FontSans, Space_Grotesk } from "next/font/google"
import { Analytics } from "@vercel/analytics/react"
import { ThemeProvider } from "next-themes"
import { SWRConfig } from "swr"
import { Toaster } from "@/components/ui/toaster"
import { toast } from "@/hooks/use-toast"
import "@/styles/globals.css"

const fontSans = Space_Grotesk({
  subsets: ["latin"],
  variable: "--font-sans",
  display: "swap",
  preload: true,
})

const fontAboretoSans = Aboreto({
  weight: "400",
  subsets: ["latin"],
  preload: true,
  variable: "--font-aboreto",
})

export default function App({ Component, pageProps }: AppProps) {
  return (
    <>
      <style jsx global>{`
        :root {
          --font-sans: ${fontSans.style.fontFamily};
          --font-aboreto: ${fontAboretoSans.style.fontFamily};
        }
      `}</style>
      <SWRConfig
        value={{
          revalidateOnFocus: false,
          onError: (error, key) => {
            toast({ title: "API Error", description: error.message })
          },
        }}
      >
        <ThemeProvider attribute="class" defaultTheme="dark" enableSystem>
          <Component {...pageProps} />
          <Analytics />
          <Toaster />
        </ThemeProvider>
      </SWRConfig>
    </>
  )
}


text:
import { useCallback, useState } from "react"
import { AnimatePresence, motion } from "framer-motion"
import { cn } from "@/lib/utils"
import { Card } from "@/components/Card"
import {
  BackgroundColorBlur,
  BackgroundGridPattern,
  PageLayout,
} from "@/components/Layouts"
import { NamespaceSelector } from "@/components/NamespaceInput"
import { DocumentQA } from "@/components/query/DocumentQA"
import { FileUpload } from "@/components/train/FileUpload"
import { UrlScraper } from "@/components/train/UrlScraper"

function ToggleHeading({ text, embedding }) {
  const activeHeading = text == embedding
  return (

      {text}

  )
}

const fadeIn = {
  hidden: { opacity: 0 },
  visible: { opacity: 1, transition: { duration: 0.35 } },
  exit: { opacity: 0, transition: { duration: 0.02 } },
}

export default function Pinecone() {
  const [namespace, setNamespace] = useState("")
  const [embedding, setEmbedding] = useState("TRAIN")
  const [animateOnce, setAnimateOnce] = useState(true)
  const handleNamespaceSelect = useCallback((selectedNamespace) => {
    console.log("Selected namespace:", selectedNamespace)
    setNamespace(selectedNamespace)
  }, [])

  function toggleEmbedding() {
    setEmbedding(embedding === "TRAIN" ? "QUERY" : "TRAIN")
    setAnimateOnce(false)
  }

  const imageVariants = {
    rotate: {
      rotateY: 90,
    },
  }

  return (
    <PageLayout>

              <button
                className={cn(
                  "flex flex-col items-center justify-center",
                  embedding === "TRAIN"
                    ? "cursor-default"
                    : "transition duration-150 hover:scale-105"
                )}
                onClick={() => setEmbedding("TRAIN")}
              >
                <ToggleHeading text="TRAIN" embedding={embedding} />
              </button>
              <button
                onClick={toggleEmbedding}
                className=" transition duration-150 hover:scale-105 md:gap-3"
              >
                <motion.img
                  className={cn(animateOnce ? "animate-pulse" : "")}
                  alt={embedding}
                  src={
                    embedding === "TRAIN"
                      ? "/logo-down-indigo.webp"
                      : "/merc-logo-down-aqua.webp"
                  }
                  variants={imageVariants}
                  animate={{ rotateX: embedding === "TRAIN" ? 0 : 180 }}
                  height={175}
                  width={175}
                />
              </button>
              <button
                className={cn(
                  "flex flex-col items-center justify-center",
                  embedding === "QUERY"
                    ? "cursor-default"
                    : "transition duration-150 hover:scale-105"
                )}
                onClick={() => setEmbedding("QUERY")}
              >
                <ToggleHeading text="QUERY" embedding={embedding} />
              </button>

              <NamespaceSelector
                newNamespace={namespace}
                onNamespaceSelect={handleNamespaceSelect}
              />

          <AnimatePresence mode="wait">
            {embedding === "TRAIN" ? (
              <motion.div
                key={"TRAIN"}
                className="  flex w-full flex-col items-center"
                initial="hidden"
                animate="visible"
                exit="exit"
                variants={fadeIn}
              >

                  Set a pinecone  namespace{" "}
                  and create vector embeddings by
                   uploading files or{" "}
                   scraped websites

                      <Card
                        cardDetails={{
                          name: "Upload",
                          description:
                            "Upload your pdf to Pinecone as a vector",
                        }}
                      >
                        <FileUpload namespace={namespace} />
                      </Card>

                      <Card
                        cardDetails={{
                          name: "Scrape",
                          description: "Scrape URLs to generate embeddings",
                        }}
                      >
                        <UrlScraper namespace={namespace} />
                      </Card>

              </motion.div>
            ) : (

                  Query the embedded knowledge provided by you.

                <DocumentQA namespace={namespace} />

            )}
          </AnimatePresence>

            <BackgroundGridPattern />

          <BackgroundColorBlur />

    </PageLayout>
  )
}


text:
import { Head, Html, Main, NextScript } from "next/document"
import { FancyBackground } from "@/components/Layouts"

export default function Document() {
  return (
    <Html lang="en">
      <Head />
      <body className="min-h-screen scroll-smooth font-sans text-zinc-900 antialiased dark:text-zinc-50">
        <FancyBackground>
          <Main />
          <NextScript />
        </FancyBackground>
      </body>
    </Html>
  )
}


text:
import type { NextApiRequest, NextApiResponse } from "next"
import { OpenAIEmbeddings } from "langchain/embeddings"
import { PineconeStore } from "langchain/vectorstores"
import { initPinecone } from "@/config/pinecone"
import { makePdfChain } from "@/lib/chain"

function prepareResponse(res: NextApiResponse) {
  res.writeHead(200, {
    "Content-Type": "text/event-stream",
    "Cache-Control": "no-cache, no-transform",
    Connection: "keep-alive",
  })
}

export default async function handler(
  req: NextApiRequest,
  res: NextApiResponse
) {
  const { namespace } = req.headers

  if (!req.body.question) {
    return res.status(400).json({ message: "No question in the request" })
  }

  const pinecone = await initPinecone()
  const index = pinecone.Index(process.env.PINECONE_INDEX_NAME)
  const namespaceConfig = !!namespace ? namespace : "default-namespace"

  const vectorStore = await PineconeStore.fromExistingIndex(
    new OpenAIEmbeddings({}),
    {
      pineconeIndex: index,
      textKey: "text",
      // @ts-ignore
      namespace: namespaceConfig,
    }
  )

  prepareResponse(res)
  await createChainAndSendResponse(req, res, vectorStore)
}

async function createChainAndSendResponse(
  req: NextApiRequest,
  res: NextApiResponse,
  vectorStore: any
) {
  const { question, history } = req.body
  const sanitizedQuestion = question.trim().replaceAll("\n", " ")

  const sendData = (data: string) => {
    res.write(`data: ${data}\n\n`)
  }

  sendData(JSON.stringify({ question: sanitizedQuestion }))
  sendData(JSON.stringify({ data: "" }))

  const chain = makePdfChain(vectorStore, (token: string) => {
    sendData(JSON.stringify({ data: token }))
  })

  try {
    const response = await chain.call({
      question: sanitizedQuestion,
      chat_history: history || [],
    })

    sendData(JSON.stringify({ sourceDocs: response.sourceDocuments }))
  } catch (error) {
    console.log("error", error)
  } finally {
    sendData("[DONE]")
    res.end()
  }
}


text:
import type { NextApiRequest, NextApiResponse } from "next"
import { OpenAIEmbeddings } from "langchain/embeddings"
import { PineconeStore } from "langchain/vectorstores"
import { Configuration, OpenAIApi } from "openai"
import { initPinecone } from "@/config/pinecone"
import { getDocumentsFromUrl } from "@/lib/webpage"

const configuration = new Configuration({ apiKey: process.env.OPENAI_API_KEY })
const openAi = new OpenAIApi(configuration)

if (
  !process.env.PINECONE_ENVIRONMENT ||
  !process.env.PINECONE_API_KEY ||
  !process.env.PINECONE_INDEX_NAME
) {
  throw new Error("Pinecone environment or api key vars missing")
}

export default async function handler(
  req: NextApiRequest,
  res: NextApiResponse
) {
  const { method, body, headers } = req
  if (method === "POST") {
    const { urls } = body
    const { namespace } = headers
    const namespaceConfig = !!namespace ? namespace : "default-namespace"

    try {
      // scrape url -> text -> Document chunk & metadata -> store in Pinecone Namespace
      const documents = await getDocumentsFromUrl(urls)

      await storeDocumentsInPinecone(documents, namespaceConfig)
      return res.status(200).json({ success: true })
    } catch (error) {
      console.log("error", error)
      throw new Error("Failed to ingest your data")
    }
  }
}

async function storeDocumentsInPinecone(docs: any, namespace) {
  const pinecone = await initPinecone()
  const embeddings = new OpenAIEmbeddings()
  const index = pinecone.Index(process.env.PINECONE_INDEX_NAME)

  await PineconeStore.fromDocuments(docs, embeddings, {
    pineconeIndex: index,
    namespace,
    textKey: "text",
  })
}


text:
import type { NextApiRequest, NextApiResponse, PageConfig } from "next"
import { OpenAIEmbeddings } from "langchain/embeddings"
import { PineconeStore } from "langchain/vectorstores"
import { initPinecone } from "@/config/pinecone"
import { getFileText, splitDocumentsFromFile } from "@/lib/file"

if (
  !process.env.PINECONE_ENVIRONMENT ||
  !process.env.PINECONE_API_KEY ||
  !process.env.PINECONE_INDEX_NAME
) {
  throw new Error("Pinecone environment or api key vars missing")
}

export async function handler(req: NextApiRequest, res: NextApiResponse) {
  const { namespace } = req.headers
  const namespaceConfig = !!namespace ? namespace : "default-namespace"

  try {
    // parse file -> text -> Document chunk & metadata -> store in Pinecone Namespace
    const fileText = await getFileText(req)
    const docs = await splitDocumentsFromFile(fileText)

    await storeDocumentsInPinecone(docs, namespaceConfig)
    res.status(200).json({ message: "Success" })
  } catch (error) {
    console.log("error", error)
    throw new Error("Failed to ingest your data")
  }
}

async function storeDocumentsInPinecone(docs: any, namespace) {
  const pinecone = await initPinecone()
  const embeddings = new OpenAIEmbeddings()
  const index = pinecone.Index(process.env.PINECONE_INDEX_NAME)

  await PineconeStore.fromDocuments(docs, embeddings, {
    pineconeIndex: index,
    namespace,
    textKey: "text",
  })
}

export const config: PageConfig = {
  api: {
    bodyParser: false,
  },
}

export default handler


text:
import type { NextApiRequest, NextApiResponse } from "next"
import { initPinecone } from "@/config/pinecone"

if (
  !process.env.PINECONE_ENVIRONMENT ||
  !process.env.PINECONE_API_KEY ||
  !process.env.PINECONE_INDEX_NAME
) {
  throw new Error("Pinecone environment or api key vars missing")
}

// Embedding document sizes
export const config = {
  runtime: "edge",
}

export default async function handler(
  req: NextApiRequest,
  res: NextApiResponse
) {
  try {
    const pinecone = await initPinecone()
    const index = pinecone.Index(process.env.PINECONE_INDEX_NAME)
    const indexDescription = await index.describeIndexStats({
      describeIndexStatsRequest: {},
    })
    const results = JSON.stringify({ indexDescription })
    // Process each document and store it in the database
    return new Response(results)
  } catch (error) {
    console.log("error", error)
    throw new Error("Failed to fetch Pinecone namespaces")
  }
}


text:
import { useState } from "react"
import { fetchEventSource } from "@microsoft/fetch-event-source"

export type Message = {
  type: "apiMessage" | "userMessage"
  message: string
  isStreaming?: boolean
  sourceDocs?: Document[]
  question?: string
}

type FileChatHook = {
  userQuestion: string
  setUserQuestion: React.Dispatch<React.SetStateAction<string>>
  status: string
  messages: Message[]
  setStatus: React.Dispatch<React.SetStateAction<string>>
  pendingSourceDocs?: Document[]
  history: [string, string][]
  answerStream?: string
  generateAnswer: (e: any) => Promise<void>
}

export function usePineconeQuery(namespace): FileChatHook {
  const [userQuestion, setUserQuestion] = useState("")
  const [status, setStatus] = useState("idle")

  const [messageState, setMessageState] = useState<{
    messages: Message[]
    answerStream?: string
    question?: string
    history: [string, string][]
    pendingSourceDocs?: Document[]
  }>({
    messages: [],
    history: [],
    pendingSourceDocs: [],
  })

  const { messages, answerStream, history, pendingSourceDocs } = messageState

  //   TODO: Make this more generic
  async function generateAnswer(e: any) {
    e.preventDefault()

    if (!userQuestion) {
      alert("Please input a question")
      return
    }

    const question = userQuestion.trim()

    setMessageState((state) => ({
      ...state,
      messages: [
        ...state.messages,
        {
          type: "userMessage",
          message: question,
        },
      ],
      answerStream: undefined,
    }))

    setUserQuestion("")
    setMessageState((state) => ({ ...state, answerStream: "" }))

    const ctrl = new AbortController()

    try {
      setStatus("loading")
      fetchEventSource("/api/query", {
        method: "POST",
        headers: {
          "Content-Type": "application/json",
          namespace: !!namespace ? namespace : "default-namespace",
        },
        body: JSON.stringify({
          question,
          history,
        }),
        signal: ctrl.signal,
        onmessage: (event) => {
          if (event.data === "[DONE]") {
            setMessageState((state) => ({
              history: [...state.history, [question, state.answerStream ?? ""]],
              messages: [
                ...state.messages,
                {
                  type: "apiMessage",
                  message: state.answerStream ?? "",
                  question: state.question ?? "",
                  sourceDocs: state.pendingSourceDocs,
                },
              ],

              answerStream: undefined,
              pendingSourceDocs: undefined,
            }))

            setStatus("complete")
            ctrl.abort()
          } else {
            const data = JSON.parse(event.data)

            if (data.sourceDocs) {
              setMessageState((state) => ({
                ...state,
                pendingSourceDocs: data.sourceDocs,
              }))
            } else if (data.question) {
              setStatus("streaming")
              setMessageState((state) => ({
                ...state,
                question: data.question,
              }))
            } else {
              setMessageState((state) => ({
                ...state,
                answerStream: (state.answerStream ?? "") + data.data,
              }))
            }
          }
        },
      })
    } catch (error) {
      setStatus("error")
      console.log("error", error)
    }
  }

  return {
    userQuestion,
    setUserQuestion,
    status,
    setStatus,
    answerStream,
    pendingSourceDocs,
    history,
    messages,
    generateAnswer,
  }
}


text:
// Inspired by react-hot-toast library
import * as React from "react"
import { ToastActionElement, type ToastProps } from "@/components/ui/toast"

const TOAST_LIMIT = 1
const TOAST_REMOVE_DELAY = 1000

type ToasterToast = ToastProps & {
  id: string
  title?: React.ReactNode
  description?: React.ReactNode
  action?: ToastActionElement
}

const actionTypes = {
  ADD_TOAST: "ADD_TOAST",
  UPDATE_TOAST: "UPDATE_TOAST",
  DISMISS_TOAST: "DISMISS_TOAST",
  REMOVE_TOAST: "REMOVE_TOAST",
} as const

let count = 0

function genId() {
  count = (count + 1) % Number.MAX_VALUE
  return count.toString()
}

type ActionType = typeof actionTypes

type Action =
  | {
      type: ActionType["ADD_TOAST"]
      toast: ToasterToast
    }
  | {
      type: ActionType["UPDATE_TOAST"]
      toast: Partial<ToasterToast>
    }
  | {
      type: ActionType["DISMISS_TOAST"]
      toastId?: ToasterToast["id"]
    }
  | {
      type: ActionType["REMOVE_TOAST"]
      toastId?: ToasterToast["id"]
    }

interface State {
  toasts: ToasterToast[]
}

const toastTimeouts = new Map<string, ReturnType<typeof setTimeout>>()

const addToRemoveQueue = (toastId: string) => {
  if (toastTimeouts.has(toastId)) {
    return
  }

  const timeout = setTimeout(() => {
    toastTimeouts.delete(toastId)
    dispatch({
      type: "REMOVE_TOAST",
      toastId: toastId,
    })
  }, TOAST_REMOVE_DELAY)

  toastTimeouts.set(toastId, timeout)
}

export const reducer = (state: State, action: Action): State => {
  switch (action.type) {
    case "ADD_TOAST":
      return {
        ...state,
        toasts: [action.toast, ...state.toasts].slice(0, TOAST_LIMIT),
      }

    case "UPDATE_TOAST":
      return {
        ...state,
        toasts: state.toasts.map((t) =>
          t.id === action.toast.id ? { ...t, ...action.toast } : t
        ),
      }

    case "DISMISS_TOAST":
      const { toastId } = action

      // ! Side effects ! - This could be extracted into a dismissToast() action,
      // but I'll keep it here for simplicity
      if (toastId) {
        addToRemoveQueue(toastId)
      } else {
        state.toasts.forEach((toast) => {
          addToRemoveQueue(toast.id)
        })
      }

      return {
        ...state,
        toasts: state.toasts.map((t) =>
          t.id === toastId || toastId === undefined
            ? {
                ...t,
                open: false,
              }
            : t
        ),
      }
    case "REMOVE_TOAST":
      if (action.toastId === undefined) {
        return {
          ...state,
          toasts: [],
        }
      }
      return {
        ...state,
        toasts: state.toasts.filter((t) => t.id !== action.toastId),
      }
  }
}

const listeners: Array<(state: State) => void> = []

let memoryState: State = { toasts: [] }

function dispatch(action: Action) {
  memoryState = reducer(memoryState, action)
  listeners.forEach((listener) => {
    listener(memoryState)
  })
}

interface Toast extends Omit<ToasterToast, "id"> {}

function toast({ ...props }: Toast) {
  const id = genId()

  const update = (props: ToasterToast) =>
    dispatch({
      type: "UPDATE_TOAST",
      toast: { ...props, id },
    })
  const dismiss = () => dispatch({ type: "DISMISS_TOAST", toastId: id })

  dispatch({
    type: "ADD_TOAST",
    toast: {
      ...props,
      id,
      open: true,
      onOpenChange: (open) => {
        if (!open) dismiss()
      },
    },
  })

  return {
    id: id,
    dismiss,
    update,
  }
}

function useToast() {
  const [state, setState] = React.useState<State>(memoryState)

  React.useEffect(() => {
    listeners.push(setState)
    return () => {
      const index = listeners.indexOf(setState)
      if (index > -1) {
        listeners.splice(index, 1)
      }
    }
  }, [state])

  return {
    ...state,
    toast,
    dismiss: (toastId?: string) => dispatch({ type: "DISMISS_TOAST", toastId }),
  }
}

export { useToast, toast }


text:
import useSWR from "swr"

export const fetcher = (...args: Parameters<typeof fetch>) =>
  fetch(...args).then((res) => res.json())

export const usePineconeStats = () => {
  const { data, error, isLoading } = useSWR("/api/stats", fetcher, {
    revalidateOnFocus: true,
  })

  return {
    loading: isLoading,
    data,
    error,
  }
}

interface IndexDescription {
  namespaces: {
    [key: string]: {
      vectorCount: number
    }
  }
  dimension: number
  indexFullness: number
  totalVectorCount: number
}

export const getNamespaceKeys = (
  indexDescription: IndexDescription
): string[] => {
  return Object.keys(indexDescription.namespaces)
}


text:
import useSWRMutation from "swr/mutation"
import { toast } from "@/hooks/use-toast"

async function sendRequest(
  url,
  { arg }: { arg: { urls: string[]; namespace: string } }
) {
  const { urls, namespace } = arg

  const response = await fetch(url, {
    method: "POST",
    headers: {
      "Content-Type": "application/json",
      namespace: !!namespace ? namespace : "default-namespace",
    },
    body: JSON.stringify({ urls }),
  })

  if (!response.ok) {
    toast({
      title: "You need to configure API Keys to use this app",
      description: response.statusText,
    })
  }

  const data = response.body

  if (!data) {
    return
  }
}

export const useScrapeEmbed = (url: string) => {
  const { trigger, error, isMutating } = useSWRMutation(
    `/api/${url}`,
    sendRequest
  )

  return {
    loading: isMutating,
    error,
    trigger,
  }
}


text:
import { useCallback, useState } from "react"

export const useToggle = (initialState: boolean = false): [boolean, any] => {
  // Initialize the state
  const [state, setState] = useState<boolean>(initialState)
  // Define and memoize toggler function in case we pass down the component,
  const toggle = useCallback((): void => setState((state) => !state), [])
  return [state, toggle]
}


text:
'use client'

import * as React from 'react'

export interface useCopyToClipboardProps {
  timeout?: number
}

export function useCopyToClipboard({
  timeout = 2000
}: useCopyToClipboardProps) {
  const [isCopied, setIsCopied] = React.useState<Boolean>(false)

  const copyToClipboard = (value: string) => {
    if (typeof window === 'undefined' || !navigator.clipboard?.writeText) {
      return
    }

    if (!value) {
      return
    }

    navigator.clipboard.writeText(value).then(() => {
      setIsCopied(true)

      setTimeout(() => {
        setIsCopied(false)
      }, timeout)
    })
  }

  return { isCopied, copyToClipboard }
}


text:
import {
  HumanMessagePromptTemplate,
  PromptTemplate,
  SystemMessagePromptTemplate,
} from "langchain/prompts"

const CONDENSE_PROMPT =
  PromptTemplate.fromTemplate(`Given the following conversation and a follow up question, rephrase the follow up question to be a standalone question.
Chat History:
{chat_history}
Follow Up Input: {question}
Standalone question:`)

const QA_PROMPT = PromptTemplate.fromTemplate(
  `You are an AI assistant providing helpful advice. You are given the following extracted parts of a long document and a question. Provide a conversational answer based on the context provided.
You should only provide hyperlinks that reference the context below. Do NOT make up hyperlinks.
If you can't find the answer in the context below, just say "Hmm, I'm not sure." Don't try to make up an answer.
If the question is not related to the context, politely respond that you are tuned to only answer questions that are related to the context.
Question: {question}
=========
{context}
=========
Answer in Markdown:`
)

const IMPROVED_CONDENSE_PROMPT = PromptTemplate.fromTemplate(
  `Your task as an AI language model is to create a clear and concise standalone question based on the given conversation history and a related follow-up question. Ensure that your rephrased question captures the essence of the follow-up question without relying on the context of the conversation.
System message: {system_message}
Conversation history:
{chat_history}
Related follow-up question: {question}
Rephrased standalone question:`
)

// - If you can't find the answer in the context, respond with "I'm sorry, but the information provided doesn't contain the answer you're looking for."
const IMPROVED_QA_PROMPT = PromptTemplate.fromTemplate(
  `As a highly advanced AI language model, your task is to provide a comprehensive and accurate response in a conversational manner, based on the context provided below. The following excerpt from a document is given, along with a question related to it. Please ensure that your answer is well-structured and directly addresses the question.
Guidelines:
- Use information from the provided context to support your answer. Do not include information from external sources.
- If the question is exactly "tl;dr" try your hardest to summarize the document in 100 words or less.
- If the question is unrelated to the context, kindly inform that your responses are limited to the information provided in the given context.

Question: {question}
=========
{context}
=========
Answer in Markdown format:`
)

const OPTIMIZED_CONDENSE_PROMPT = PromptTemplate.fromTemplate(
  `As an AI language model, your task is to create a clear, concise, and effective standalone question based on the conversation history and a related follow-up question provided below. Make sure your rephrased question captures the core intent of the follow-up question without relying on the conversation context.
Conversation history:
{chat_history}
Related follow-up question: {question}
Rephrased standalone question:`
)

const OPTIMIZED_QA_PROMPT = PromptTemplate.fromTemplate(
  `As an advanced AI language model, your goal is to provide a detailed, accurate, and conversational response to a question based on the context from a document excerpt provided below. Ensure your answer is well-structured and directly addresses the question.
System message: {system_message}
Guidelines:
- Rely solely on the provided context for your answer. Do not include information from external sources.
- If the answer cannot be found in the context, respond with "Unfortunately, the information provided doesn't contain the answer you're looking for."
- If the question is unrelated to the context, kindly mention that your ability to respond is limited to the context provided.

Question: {question}
=========
{context}
=========
Answer in Markdown format:`
)

export {
  CONDENSE_PROMPT,
  IMPROVED_CONDENSE_PROMPT,
  OPTIMIZED_CONDENSE_PROMPT,
  OPTIMIZED_QA_PROMPT,
  QA_PROMPT,
  IMPROVED_QA_PROMPT,
}


text:
import * as cheerio from "cheerio"
import { Document } from "langchain/document"
import { RecursiveCharacterTextSplitter } from "langchain/text_splitter"

async function splitDocumentsFromUrl(pageContent: string, url) {
  const rawDocs = new Document({
    pageContent,
    metadata: { source: url, type: "scrape" },
  })
  const textSplitter = new RecursiveCharacterTextSplitter({
    chunkSize: 1000,
    chunkOverlap: 200,
  })
  const docs = await textSplitter.splitDocuments([rawDocs])

  return docs
}

export async function getDocumentsFromUrl(urls: string[]) {
  const documents = []

  // Fetch and process the content of each URL
  for (const url of urls) {
    const response = await fetch(url)
    const html = await response.text()
    const $ = cheerio.load(html)
    const text = $("body").text()

    const chunks = await splitDocumentsFromUrl(text, url)

    // Add the chunks to the documents array
    chunks.forEach((chunk) => documents.push(chunk))
  }

  return documents
}


text:
import type { NextApiRequest } from "next"
import formidable from "formidable"
import { Document } from "langchain/document"
import { RecursiveCharacterTextSplitter } from "langchain/text_splitter"
import mammoth from "mammoth"
import { NodeHtmlMarkdown } from "node-html-markdown"
import pdfParse from "pdf-parse"
import { Writable } from "stream"
import Tesseract from "tesseract.js"

const formidableConfig = {
  keepExtensions: true,
  maxFileSize: 20_000_000,
  maxFieldsSize: 30_000_000,
  maxFields: 7,
  allowEmptyFiles: false,
  multiples: false,
}

export const getTextContentFromPDF = async (pdfBuffer) => {
  // TODO: pass metadata
  const { text, metadata } = await pdfParse(pdfBuffer)
  console.log("metadata", metadata)
  return text
}

export async function extractTextFromImage(imagePath) {
  return Tesseract.recognize(imagePath, "eng", {
    logger: (m) => console.log(m),
  }).then(({ data: { text } }) => {
    console.log(text)
    return text
  })
}

export const formidablePromise = (
  req: NextApiRequest,
  opts?: Parameters<typeof formidable>[0]
): Promise<{ fields: formidable.Fields; files: formidable.Files }> => {
  return new Promise((accept, reject) => {
    const form = formidable(opts)

    form.parse(req, (err, fields, files) => {
      if (err) {
        return reject(err)
      }
      return accept({ fields, files })
    })
  })
}

export const fileConsumer = <T = unknown>(acc: T[]) => {
  const writable = new Writable({
    write: (chunk, _enc, next) => {
      acc.push(chunk)
      next()
    },
  })

  return writable
}

const convertFileToString = async (file: formidable.File, chunks) => {
  const fileData = Buffer.concat(chunks)

  let fileText = ""
  let docs

  switch (file.mimetype) {
    case "application/pdf":
      fileText = await getTextContentFromPDF(fileData)
      // docs = await createDocumentsFromPDFFile(file)
      break
    case "application/vnd.openxmlformats-officedocument.wordprocessingml.document": // i.e. docx file
      const docxResult = await mammoth.extractRawText({
        buffer: fileData,
      })
      fileText = docxResult.value
      break
    // case "application/vnd.openxmlformats-officedocument.presentationml.presentation": // i.e. pptx file
    //   const pptResult = await extractTextFromPPTX(file.filepath)
    //   console.log(pptResult)
    //   fileText = pptResult.toString()
    //   break
    case "application/octet-stream":
      fileText = fileData.toString()
      break
    case "application/json":
      fileText = await fileData.toString()
      break
    case "text/markdown":
      fileText = await fileData.toString()
      break
    case "text/csv":
      break
    case "image/jpeg":
      fileText = await extractTextFromImage(fileData)
      break
    case "image/png":
      fileText = await extractTextFromImage(fileData)
      break
    case "text/html":
      const html = fileData.toString()
      const translatedHtml = NodeHtmlMarkdown.translate(html)
      fileText = translatedHtml
      break
    case "text/plain":
      fileText = fileData.toString()
      break
    default:
      throw new Error("Unsupported file type")
  }

  return {
    fileText,
    docs,
    fileName: file.originalFilename ?? "fallback-filename",
  }
}

export const getFileText = async (req: NextApiRequest) => {
  const chunks: never[] = []
  const { fields, files } = await formidablePromise(req, {
    ...formidableConfig,
    // consume this, otherwise formidable tries to save the file to disk
    fileWriteStreamHandler: () => fileConsumer(chunks),
  })

  const { file } = files

  return convertFileToString(file as formidable.File, chunks)
}

export const splitDocumentsFromFile = async (file) => {
  const { fileText, fileName } = file

  const textSplitter = new RecursiveCharacterTextSplitter({
    chunkSize: 1000,
    chunkOverlap: 200,
  })
  const docOutput = await textSplitter.splitDocuments([
    new Document({
      pageContent: fileText,
      metadata: { source: fileName, type: "file" },
    }),
  ])

  return docOutput
}


text:
import { ClassValue, clsx } from "clsx"
import { twMerge } from "tailwind-merge"

export function cn(...inputs: ClassValue[]) {
  return twMerge(clsx(inputs))
}

export function pluralize(
  word: string,
  count: number,
  pluralForm?: string
): string {
  if (count === 1) {
    return word
  }

  if (pluralForm) {
    return pluralForm
  }

  return `${word}s`
}

function isValidUrl(urlString: string): boolean {
  try {
    new URL(urlString)
    return true
  } catch {
    return false
  }
}

function removeLeadingDash(pathPart: string): string {
  return pathPart.startsWith(` - `) ? pathPart.substring(1) : pathPart
}

function getSecondLevelDomain(hostname: string): string {
  return hostname.split(".").slice(-2, -1).join(".")
}

function removeFileExtension(pathPart: string | undefined): string {
  if (!pathPart) return ""
  return pathPart.replace(/\.[^/.]+$/, "")
}

function shortenLastPathPart(pathPart: string, maxLength: number): string {
  if (!pathPart) return ""
  return pathPart.length > maxLength
    ? pathPart.substring(0, maxLength)
    : pathPart
}

export function truncateLongUrl(
  longUrl: string,
  maxLength: number = 15
): string {
  if (!isValidUrl(longUrl)) return "Invalid URL"

  const cleanUrl = removeLeadingDash(longUrl)
  const url = new URL(cleanUrl)
  const hostname = getSecondLevelDomain(url.hostname)
  const pathParts = url.pathname.split("/").filter((part) => part !== "")
  let lastPathPart = pathParts[pathParts.length - 1]

  lastPathPart = removeFileExtension(lastPathPart)
  lastPathPart = shortenLastPathPart(lastPathPart, maxLength)

  const shortUrl = `${hostname}${lastPathPart ? "/" : ""}${lastPathPart}`
  //   return shortUrl
  return shortUrl.replace(/-/g, "")
}

export function truncateLongFileName(
  fileName: string,
  maxLength: number = 15
): string {
  const fileBaseName = fileName.replace(/\.[^/.]+$/, "") // Remove file extension
  const hyphenatedName = fileBaseName.replace(/\W+/g, "-") // Replace non-word characters with hyphens
  const truncatedName = hyphenatedName.slice(0, maxLength) // Truncate to the specified maxLength
  const cleanTruncatedName = truncatedName.replace(/-$/, "") // Remove trailing hyphen if present

  // Add back the file extension
  const fileExtension = fileName.match(/\.[^/.]+$/) || [""]
  const truncatedFileName = `${cleanTruncatedName}${fileExtension[0]}`

  return truncatedFileName
}

// A function that takes a file name and a string and returns true if the file name is contained in the string
// after removing punctuation and whitespace from both
export const isFileNameInString = (fileName: string, str: string) => {
  // Check if the input string is null or undefined
  if (!str) {
    return false
  }

  // Convert both to lowercase and remove punctuation and whitespace
  const normalizedFileName = fileName
    .toLowerCase()
    .replace(/[.,/#!$%^&*;:{}=-_~()\s]/g, "")
  const normalizedStr = str
    .toLowerCase()
    .replace(/[.,/#!$%^&*;:{}=-_~()\s]/g, "")

  // Return true if the normalized file name is included in the normalized string
  return normalizedStr.includes(normalizedFileName)
}


text:
import { CallbackManager } from "langchain/callbacks"
import { ChatVectorDBQAChain, LLMChain, loadQAChain } from "langchain/chains"
import { OpenAIChat } from "langchain/llms/openai"
import { PineconeStore } from "langchain/vectorstores/pinecone"
import { IMPROVED_QA_PROMPT, OPTIMIZED_CONDENSE_PROMPT } from "@/lib/prompts"

export const makePdfChain = (
  vectorstore: PineconeStore,
  onTokenStream?: (token: string) => void,
  sourceCount?: number
) => {
  const questionGenerator = new LLMChain({
    llm: new OpenAIChat({ temperature: 0 }),
    prompt: OPTIMIZED_CONDENSE_PROMPT,
  })
  const docChain = loadQAChain(
    new OpenAIChat({
      temperature: 0,
      modelName: "gpt-4", //change this to older versions (e.g. gpt-3.5-turbo) if you don't have access to gpt-4
      streaming: Boolean(onTokenStream),
      callbackManager: onTokenStream
        ? CallbackManager.fromHandlers({
            async handleLLMNewToken(token) {
              onTokenStream(token)
            },
          })
        : undefined,
    }),
    { prompt: IMPROVED_QA_PROMPT }
  )

  return new ChatVectorDBQAChain({
    vectorstore,
    combineDocumentsChain: docChain,
    questionGeneratorChain: questionGenerator,
    returnSourceDocuments: true,
    k: !!sourceCount ? sourceCount : 6  , //number of source documents to return
  })
}


text:
"use client"

import * as React from "react"
import { useRouter } from "next/navigation"
import { DialogProps } from "@radix-ui/react-alert-dialog"
import { Circle, Laptop, Moon, SunMedium } from "lucide-react"
import { useTheme } from "next-themes"
import { siteConfig } from "@/config/site"
import { cn } from "@/lib/utils"
import { Button } from "@/components/ui/button"
import {
  CommandDialog,
  CommandEmpty,
  CommandGroup,
  CommandInput,
  CommandItem,
  CommandList,
  CommandSeparator,
} from "@/components/ui/command"

export function CommandMenu({ ...props }: DialogProps) {
  const router = useRouter()
  const [open, setOpen] = React.useState(false)
  const { setTheme } = useTheme()

  React.useEffect(() => {
    const down = (e: KeyboardEvent) => {
      if (e.key === "k" && (e.metaKey || e.ctrlKey)) {
        e.preventDefault()
        setOpen((open) => !open)
      }
    }

    document.addEventListener("keydown", down)
    return () => document.removeEventListener("keydown", down)
  }, [])

  const runCommand = React.useCallback((command: () => unknown) => {
    setOpen(false)
    command()
  }, [])

  return (
    <>
      <Button
        variant="outline"
        className={cn(
          "relative h-9 w-full justify-start border-mauve-8 font-sans text-sm text-mauve-11 sm:pr-12 md:w-40 lg:w-64"
        )}
        onClick={() => setOpen(true)}
        {...props}
      >
        Create...
        Search...

          ⌘K

      </Button>
      <CommandDialog open={open} onOpenChange={setOpen}>
        <CommandInput placeholder="Type a command or search..." />
        <CommandList>
          <CommandEmpty>No results found.</CommandEmpty>

          <CommandGroup heading="Components">
            {siteConfig.mainNav.map((page) => (
              <CommandItem
                key={page.title}
                onSelect={() => {
                  runCommand(() => router.push(page.href))
                }}
              >

                  <Circle className="h-3 w-3" />

                {page.title}
              </CommandItem>
            ))}
          </CommandGroup>
          <CommandSeparator />
          <CommandGroup heading="Theme">
            <CommandItem onSelect={() => runCommand(() => setTheme("light"))}>
              <SunMedium className="mr-2 h-4 w-4" />
              Light
            </CommandItem>
            <CommandItem onSelect={() => runCommand(() => setTheme("dark"))}>
              <Moon className="mr-2 h-4 w-4" />
              Dark
            </CommandItem>
            <CommandItem onSelect={() => runCommand(() => setTheme("system"))}>
              <Laptop className="mr-2 h-4 w-4" />
              System
            </CommandItem>
          </CommandGroup>
        </CommandList>
      </CommandDialog>
    </>
  )
}


text:
import * as React from "react"
import Image from "next/image"
import { useTheme } from "next-themes"
import { NavHeader } from "@/components/NavHeader"
import { Icons } from "@/components/icons"
import { Button } from "@/components/ui/button"
import {
  DropdownMenu,
  DropdownMenuContent,
  DropdownMenuItem,
  DropdownMenuTrigger,
} from "@/components/ui/dropdown-menu"

interface LayoutProps {
  children: React.ReactNode
}

function PageLayout({ children }: LayoutProps) {
  return (
    <>
      <NavHeader />

{children}

    </>
  )
}

function Logo({ path, width, height, rotate = false }) {
  return (

      <Image
        className={`${rotate ? "rotate-[180deg]" : ""}`}
        height={height}
        width={width}
        src={path}
        alt="logo"
      />

  )
}

function FancyBackground({ children }) {
  return (

      {children}

  )
}

function BackgroundGridPattern() {
  return (
    <svg
      className="absolute inset-0 -z-10 h-full w-full stroke-neutral-500/50 [mask-image:radial-gradient(100%_100%_at_top_right,white,transparent)]"
      aria-hidden="true"
    >
      <defs>
        <pattern
          id="83fd4e5a-9d52-42fc-97b6-718e5d7ee527"
          width={200}
          height={200}
          x="50%"
          y={-1}
          patternUnits="userSpaceOnUse"
        >
          <path d="M100 200V.5M.5 .5H200" fill="none" />
        </pattern>
      </defs>
      <svg
        x="50%"
        y={-1}
        className="overflow-visible fill-neutral-200/30 dark:fill-neutral-800/30"
      >
        <path
          d="M-100.5 0h201v201h-201Z M699.5 0h201v201h-201Z M499.5 400h201v201h-201Z M-300.5 600h201v201h-201Z"
          strokeWidth={0}
        />
      </svg>
      <rect
        width="100%"
        height="100%"
        strokeWidth={0}
        fill="url(#83fd4e5a-9d52-42fc-97b6-718e5d7ee527)"
      />
    </svg>
  )
}

function BackgroundColorBlur() {
  return (

  )
}

function ThemeToggle() {
  let { theme, setTheme } = useTheme()

  return (
    <DropdownMenu>
      <DropdownMenuTrigger asChild>
        <Button variant="ghost" size="sm">
          <Icons.sun className="rotate-0 scale-100 transition-all hover:text-zinc-900 dark:-rotate-90 dark:scale-0 dark:text-zinc-400 dark:hover:text-zinc-100" />
          <Icons.moon className="absolute rotate-90 scale-0 transition-all hover:text-zinc-900 dark:rotate-0 dark:scale-100 dark:text-zinc-400 dark:hover:text-zinc-100" />
          Toggle theme
        </Button>
      </DropdownMenuTrigger>
      <DropdownMenuContent align="end" forceMount>
        <DropdownMenuItem onClick={() => setTheme("light")}>
          <Icons.sun className="mr-2 h-4 w-4" />
          Light
        </DropdownMenuItem>
        <DropdownMenuItem onClick={() => setTheme("dark")}>
          <Icons.moon className="mr-2 h-4 w-4" />
          Dark
        </DropdownMenuItem>
        <DropdownMenuItem onClick={() => setTheme("system")}>
          <Icons.laptop className="mr-2 h-4 w-4" />
          System
        </DropdownMenuItem>
      </DropdownMenuContent>
    </DropdownMenu>
  )
}

export {
  PageLayout,
  Logo,
  FancyBackground,
  ThemeToggle,
  BackgroundGridPattern,
  BackgroundColorBlur,
}


text:
import { FC,  memo } from "react"
import ReactMarkdown, {Options} from "react-markdown"

export const MemoizedReactMarkdown: FC<Options> = memo(
  ReactMarkdown,
  (prevProps, nextProps) =>
    prevProps.children === nextProps.children &&
    prevProps.className === nextProps.className
)


text:
import React, { useState } from "react"
import { cn } from "@/lib/utils"
import { Input } from "@/components/ui/input"
import { Label } from "@/components/ui/label"
import {
  Select,
  SelectContent,
  SelectItem,
  SelectTrigger,
  SelectValue,
} from "@/components/ui/select"
import { getNamespaceKeys, usePineconeStats } from "@/hooks/use-pinecone-stats"

export function NamespaceSelector({ newNamespace, onNamespaceSelect }) {
  const [namespace, setNamespace] = useState(newNamespace)

  const { data, error } = usePineconeStats()

  const handleInputChange = (event) => {
    const inputValue = event.target.value
    setNamespace(inputValue)
    onNamespaceSelect(inputValue)
  }

  const handleDropdownChange = (selectedValue) => {
    setNamespace(selectedValue)
    onNamespaceSelect(selectedValue)
  }

  const previousNamespaces =
    !error && data && getNamespaceKeys(data.indexDescription)

  return (

          namespace

        <Input
          value={namespace}
          onChange={handleInputChange}
          placeholder="default"
        />

          previous

        <Select onValueChange={handleDropdownChange}>
          <SelectTrigger
            className={cn(
              " input-shadow h-13 w-[250px] rounded-lg !outline-none",
              "relative border border-black/5 bg-white px-7 py-3.5 text-base shadow-black/5  placeholder:text-neutral-400 ",
              " dark:bg-black/80 dark:text-white dark:shadow-black/10 dark:placeholder:text-neutral-300 dark:focus:bg-black"
            )}
          >
            <SelectValue placeholder="default" />
          </SelectTrigger>

          <SelectContent>
            {previousNamespaces &&
              previousNamespaces.map((namespace, i) => (
                <SelectItem key={`${namespace}-${i}`} value={namespace}>
                  {namespace}
                </SelectItem>
              ))}
          </SelectContent>
        </Select>

  )
}


text:
import { motion } from "framer-motion"
import {
  ArrowDownRight,
  Globe,
  Laptop,
  LayoutList,
  Link,
  Loader,
  Loader2,
  type Icon as LucideIcon,
  LucideProps,
  Moon,
  SunMedium,
  Twitter,
  UploadCloud,
} from "lucide-react"
import { cn } from "@/lib/utils"

export type Icon = LucideIcon

let x = 1
const t = (v) => x * v

let checkIconTransition = {
  ease: "easeOut",
  type: "tween",
  delay: t(0.2),
  duration: t(0.3),
}
let checkIconVariants = {
  complete: {
    pathLength: [0, 1],
  },
}

export const Icons = {
  sun: SunMedium,
  moon: Moon,
  laptop: Laptop,
  twitter: Twitter,
  link: Link,
  arrowDR: ArrowDownRight,
  upload: UploadCloud,
  web: Globe,
  loading: Loader2,
  list: LayoutList,
  loadingSpinner: Loader,

  gitHub: (props: LucideProps) => (
    <svg viewBox="0 0 438.549 438.549" {...props}>
      <path
        fill="currentColor"
        d="M409.132 114.573c-19.608-33.596-46.205-60.194-79.798-79.8-33.598-19.607-70.277-29.408-110.063-29.408-39.781 0-76.472 9.804-110.063 29.408-33.596 19.605-60.192 46.204-79.8 79.8C9.803 148.168 0 184.854 0 224.63c0 47.78 13.94 90.745 41.827 128.906 27.884 38.164 63.906 64.572 108.063 79.227 5.14.954 8.945.283 11.419-1.996 2.475-2.282 3.711-5.14 3.711-8.562 0-.571-.049-5.708-.144-15.417a2549.81 2549.81 0 01-.144-25.406l-6.567 1.136c-4.187.767-9.469 1.092-15.846 1-6.374-.089-12.991-.757-19.842-1.999-6.854-1.231-13.229-4.086-19.13-8.559-5.898-4.473-10.085-10.328-12.56-17.556l-2.855-6.57c-1.903-4.374-4.899-9.233-8.992-14.559-4.093-5.331-8.232-8.945-12.419-10.848l-1.999-1.431c-1.332-.951-2.568-2.098-3.711-3.429-1.142-1.331-1.997-2.663-2.568-3.997-.572-1.335-.098-2.43 1.427-3.289 1.525-.859 4.281-1.276 8.28-1.276l5.708.853c3.807.763 8.516 3.042 14.133 6.851 5.614 3.806 10.229 8.754 13.846 14.842 4.38 7.806 9.657 13.754 15.846 17.847 6.184 4.093 12.419 6.136 18.699 6.136 6.28 0 11.704-.476 16.274-1.423 4.565-.952 8.848-2.383 12.847-4.285 1.713-12.758 6.377-22.559 13.988-29.41-10.848-1.14-20.601-2.857-29.264-5.14-8.658-2.286-17.605-5.996-26.835-11.14-9.235-5.137-16.896-11.516-22.985-19.126-6.09-7.614-11.088-17.61-14.987-29.979-3.901-12.374-5.852-26.648-5.852-42.826 0-23.035 7.52-42.637 22.557-58.817-7.044-17.318-6.379-36.732 1.997-58.24 5.52-1.715 13.706-.428 24.554 3.853 10.85 4.283 18.794 7.952 23.84 10.994 5.046 3.041 9.089 5.618 12.135 7.708 17.705-4.947 35.976-7.421 54.818-7.421s37.117 2.474 54.823 7.421l10.849-6.849c7.419-4.57 16.18-8.758 26.262-12.565 10.088-3.805 17.802-4.853 23.134-3.138 8.562 21.509 9.325 40.922 2.279 58.24 15.036 16.18 22.559 35.787 22.559 58.817 0 16.178-1.958 30.497-5.853 42.966-3.9 12.471-8.941 22.457-15.125 29.979-6.191 7.521-13.901 13.85-23.131 18.986-9.232 5.14-18.182 8.85-26.84 11.136-8.662 2.286-18.415 4.004-29.263 5.146 9.894 8.562 14.842 22.077 14.842 40.539v60.237c0 3.422 1.19 6.279 3.572 8.562 2.379 2.279 6.136 2.95 11.276 1.995 44.163-14.653 80.185-41.062 108.068-79.226 27.88-38.161 41.825-81.126 41.825-128.906-.01-39.771-9.818-76.454-29.414-110.049z"
      ></path>
    </svg>
  ),
  mercury: (props: LucideProps) => (
    <svg
      xmlns="http://www.w3.org/2000/svg"
      width="312px"
      height="239px"
      viewBox="0 0 612 539"
      version="1.0"
      stroke="#09342e"
      id="svg1339"
      {...props}
    >
      <defs id="defs1342" />
      <path
        d="M 14.456625,0.0044630784 C -0.25689377,3.3420931 -1.7502908,21.255463 1.3162792,33.468863 C 3.6956699,51.279573 17.106686,68.926373 9.0218199,87.057883 C -3.6356168,112.64337 -2.2623188,143.46795 10.624403,168.67822 C 23.17246,195.43429 47.209701,216.3602 75.297732,225.33506 C 87.612972,231.81909 80.686322,249.48487 66.955952,245.1411 C 54.081552,245.46272 35.84791,244.17577 29.73708,258.33467 C 23.529089,274.04901 42.140506,282.8172 55.017132,283.33294 C 68.829742,282.31642 87.049612,283.86994 81.760692,302.90336 C 82.287312,314.58496 81.709262,333.56426 96.680419,335.91366 C 110.56671,339.06466 118.01385,325.29256 117.81069,313.21016 C 119.06239,302.02256 114.1895,280.43597 132.47949,283.87235 C 145.95367,283.76225 163.96136,283.03934 170.74189,269.28898 C 175.49053,253.64417 158.14737,244.65819 144.87809,245.39849 C 135.82885,244.3699 116.05773,248.83439 118.48618,233.83475 C 124.3543,219.22433 144.67174,221.31388 154.7829,210.22146 C 189.76836,185.08265 210.67872,136.27985 194.08514,94.726473 C 186.69175,81.975503 187.22425,66.898443 193.13303,53.728113 C 197.47342,38.907583 204.60459,21.328723 195.83018,6.7943631 C 186.16079,-7.9576469 168.7184,4.8689031 166.19868,18.090233 C 162.34593,29.740993 161.58023,50.222533 144.55628,38.384783 C 117.21546,25.941423 84.822872,25.290213 57.350792,37.717933 C 47.696329,42.956483 37.027609,43.782883 37.322425,29.779723 C 34.056703,18.278763 29.598244,-0.46608692 14.456625,0.0044630784 z M 111.03847,63.839023 C 142.71746,68.533963 166.26701,99.940933 163.45116,131.53816 C 162.471,159.34504 140.65041,182.99938 114.13875,189.50706 C 90.406072,196.3156 64.088442,184.30613 49.164372,165.66873 C 25.576527,135.98382 36.237279,86.693763 70.553582,70.094533 C 82.926732,63.413093 97.277729,61.986873 111.03847,63.839023 z "
        id="path1374"
      />
    </svg>
  ),
  check: ({ className, ...props }) => (
    <svg
      fill="none"
      viewBox="0 0 24 24"
      className={cn(
        "-ml-0.5 h-7 w-7 text-teal-400/80 group-hover:text-teal-500  dark:text-teal-400 dark:group-hover:text-teal-300",
        className
      )}
      stroke="currentColor"
      strokeWidth={3}
      {...props}
    >
      <motion.path
        variants={checkIconVariants}
        transition={checkIconTransition}
        strokeLinecap="round"
        strokeLinejoin="round"
        d="M5 13l4 4L19 7"
      />
    </svg>
  ),
  plus: ({ className, ...props }) => (
    <svg
      {...props}
      xmlns="http://www.w3.org/2000/svg"
      className={cn(
        "-ml-0.5 h-7 w-7 text-teal-400/80 group-hover:text-teal-500  dark:text-teal-400 dark:group-hover:text-teal-300",
        className
      )}
      width="24"
      height="24"
      viewBox="0 0 24 24"
      fill="none"
      stroke="currentColor"
      strokeWidth="2"
      strokeLinecap="round"
      strokeLinejoin="round"
    >
      <motion.line
        variants={checkIconVariants}
        transition={checkIconTransition}
        x1="12"
        y1="5"
        x2="12"
        y2="19"
      ></motion.line>
      <motion.line
        variants={checkIconVariants}
        transition={checkIconTransition}
        x1="5"
        y1="12"
        x2="19"
        y2="12"
      ></motion.line>
    </svg>
  ),
}