/*
 * tokenizer.c
 * Copyright (C) 2014 Kovid Goyal <kovid at kovidgoyal.net>
 *
 * Distributed under terms of the GPL3 license.
 */

#define UNICODE
#define PY_SSIZE_T_CLEAN
#include <Python.h>
#include <structmember.h>

// Token type definition {{{
typedef struct {
    PyObject_HEAD
        // Type-specific fields go here.
        PyObject *is_container;
    PyObject *type;
    PyObject *_as_css;
    PyObject *value;
    PyObject *unit;
    PyObject *line;
    PyObject *column;

} tokenizer_Token;

static void
tokenizer_Token_dealloc(tokenizer_Token *self) {
    Py_XDECREF(self->is_container);
    self->is_container = NULL;
    Py_XDECREF(self->type);
    self->type = NULL;
    Py_XDECREF(self->_as_css);
    self->_as_css = NULL;
    Py_XDECREF(self->value);
    self->value = NULL;
    Py_XDECREF(self->unit);
    self->unit = NULL;
    Py_XDECREF(self->line);
    self->line = NULL;
    Py_XDECREF(self->column);
    self->column = NULL;
    Py_TYPE(self)->tp_free((PyObject *)self);
}


static PyObject *
tokenizer_Token_new(PyTypeObject *type, PyObject *args, PyObject *kwds) {
    tokenizer_Token *self = NULL;
    self = (tokenizer_Token *)type->tp_alloc(type, 0);
    if (self == NULL) return PyErr_NoMemory();

    if (!PyArg_ParseTuple(args, "OOOOOO", &(self->type), &(self->_as_css), &(self->value), &(self->unit), &(self->line), &(self->column))) {
        Py_TYPE(self)->tp_free((PyObject *)self);
        return NULL;
    }
    Py_INCREF(self->type);
    Py_INCREF(self->_as_css);
    Py_INCREF(self->value);
    Py_INCREF(self->unit);
    Py_INCREF(self->line);
    Py_INCREF(self->column);
    self->is_container = Py_False;
    Py_INCREF(self->is_container);

    return (PyObject *)self;
}

static PyObject *
tokenizer_Token_repr(tokenizer_Token *self) {
    PyObject *type = NULL, *line = NULL, *column = NULL, *value = NULL, *ans = NULL, *unit = NULL;
    if (!self->type || !self->line || !self->column || !self->value) return PyBytes_FromString("<Token NULL fields>");
    type = PyObject_Str(self->type);
    line = PyObject_Str(self->line);
    column = PyObject_Str(self->column);
    value = PyObject_Str(self->value);
    if (type && line && column && value) {
        if (self->unit != NULL && PyObject_IsTrue(self->unit)) {
            unit = PyObject_Str(self->unit);
            if (unit != NULL) ans = PyUnicode_FromFormat("<Token %U at %U:%U %U%U>", type, line, column, value, unit);
            else PyErr_NoMemory();
        } else ans = PyUnicode_FromFormat("<Token %U at %U:%U %U>", type, line, column, value);
    } else PyErr_NoMemory();
    Py_XDECREF(type);
    Py_XDECREF(line);
    Py_XDECREF(column);
    Py_XDECREF(value);
    Py_XDECREF(unit);
    return ans;
}

static PyObject *
tokenizer_Token_as_css(tokenizer_Token *self, PyObject *args, PyObject *kwargs) {
    if (!self->_as_css) { Py_RETURN_NONE; }
    Py_INCREF(self->_as_css);
    return self->_as_css;
}

static PyMemberDef tokenizer_Token_members[] = {
    {"is_container", T_OBJECT_EX, offsetof(tokenizer_Token, is_container), 0, "False unless this token is a  container for other tokens"},
    {"type", T_OBJECT_EX, offsetof(tokenizer_Token, type), 0, "The token type"},
    {"_as_css", T_OBJECT_EX, offsetof(tokenizer_Token, _as_css), 0, "Internal variable, use as_css() method instead."},
    {"value", T_OBJECT_EX, offsetof(tokenizer_Token, value), 0, "The token value"},
    {"unit", T_OBJECT_EX, offsetof(tokenizer_Token, unit), 0, "The token unit"},
    {"line", T_OBJECT_EX, offsetof(tokenizer_Token, line), 0, "The token line number"},
    {"column", T_OBJECT_EX, offsetof(tokenizer_Token, column), 0, "The token column number"},
    {NULL} /* Sentinel */
};

static PyMethodDef tokenizer_Token_methods[] = {
    {"as_css", (PyCFunction)tokenizer_Token_as_css, METH_VARARGS, "as_css() -> Return the CSS representation of this token"},

    {NULL} /* Sentinel */
};

static PyTypeObject tokenizer_TokenType = {
    // {{{
    PyVarObject_HEAD_INIT(NULL, 0)
    /* tp_name           */ "tokenizer.Token",
    /* tp_basicsize      */ sizeof(tokenizer_Token),
    /* tp_itemsize       */ 0,
    /* tp_dealloc        */ (destructor)tokenizer_Token_dealloc,
    /* tp_print          */ 0,
    /* tp_getattr        */ 0,
    /* tp_setattr        */ 0,
    /* tp_compare        */ 0,
    /* tp_repr           */ (reprfunc)tokenizer_Token_repr,
    /* tp_as_number      */ 0,
    /* tp_as_sequence    */ 0,
    /* tp_as_mapping     */ 0,
    /* tp_hash           */ 0,
    /* tp_call           */ 0,
    /* tp_str            */ 0,
    /* tp_getattro       */ 0,
    /* tp_setattro       */ 0,
    /* tp_as_buffer      */ 0,
    /* tp_flags          */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
    /* tp_doc            */ "Token",
    /* tp_traverse       */ 0,
    /* tp_clear          */ 0,
    /* tp_richcompare    */ 0,
    /* tp_weaklistoffset */ 0,
    /* tp_iter           */ 0,
    /* tp_iternext       */ 0,
    /* tp_methods        */ tokenizer_Token_methods,
    /* tp_members        */ tokenizer_Token_members,
    /* tp_getset         */ 0,
    /* tp_base           */ 0,
    /* tp_dict           */ 0,
    /* tp_descr_get      */ 0,
    /* tp_descr_set      */ 0,
    /* tp_dictoffset     */ 0,
    /* tp_init           */ 0,
    /* tp_alloc          */ 0,
    /* tp_new            */ tokenizer_Token_new,
}; // }}}
// }}}

static PyObject *COMPILED_TOKEN_REGEXPS = NULL, *UNICODE_UNESCAPE = NULL, *NEWLINE_UNESCAPE = NULL, *SIMPLE_UNESCAPE = NULL, *FIND_NEWLINES = NULL,
                *TOKEN_DISPATCH = NULL;
static PyObject *COLON = NULL, *SCOLON = NULL, *LPAR = NULL, *RPAR = NULL, *LBRACE = NULL, *RBRACE = NULL, *LBOX = NULL, *RBOX = NULL, *DELIM_TOK = NULL,
                *INTEGER = NULL, *STRING_TOK = NULL;

static Py_ssize_t BAD_COMMENT, BAD_STRING, PERCENTAGE, DIMENSION, ATKEYWORD, FUNCTION, COMMENT, NUMBER, STRING, IDENT, HASH, URI, DELIM = -1;

#define CLEANUP(x)   \
    Py_XDECREF((x)); \
    x = NULL;

static PyObject *
tokenize_cleanup(PyObject *self, PyObject *args) {
    CLEANUP(COMPILED_TOKEN_REGEXPS);
    CLEANUP(UNICODE_UNESCAPE);
    CLEANUP(NEWLINE_UNESCAPE);
    CLEANUP(SIMPLE_UNESCAPE);
    CLEANUP(FIND_NEWLINES);
    CLEANUP(TOKEN_DISPATCH);
    CLEANUP(COLON);
    CLEANUP(SCOLON);
    CLEANUP(LPAR);
    CLEANUP(RPAR);
    CLEANUP(LBRACE);
    CLEANUP(RBRACE);
    CLEANUP(LBOX);
    CLEANUP(RBOX);
    CLEANUP(DELIM_TOK);
    CLEANUP(INTEGER);
    CLEANUP(STRING_TOK);
    Py_RETURN_NONE;
}

static PyObject *
tokenize_init(PyObject *self, PyObject *args) {
    PyObject *cti = NULL;

    if (COMPILED_TOKEN_REGEXPS != NULL) { tokenize_cleanup(NULL, NULL); }
    if (!PyArg_ParseTuple(
            args,
            "OOOOOOOOOOOOOOOOOO",
            &COMPILED_TOKEN_REGEXPS,
            &UNICODE_UNESCAPE,
            &NEWLINE_UNESCAPE,
            &SIMPLE_UNESCAPE,
            &FIND_NEWLINES,
            &TOKEN_DISPATCH,
            &cti,
            &COLON,
            &SCOLON,
            &LPAR,
            &RPAR,
            &LBRACE,
            &RBRACE,
            &LBOX,
            &RBOX,
            &DELIM_TOK,
            &INTEGER,
            &STRING_TOK))
        return NULL;
    Py_INCREF(COMPILED_TOKEN_REGEXPS);
    Py_INCREF(UNICODE_UNESCAPE);
    Py_INCREF(NEWLINE_UNESCAPE);
    Py_INCREF(SIMPLE_UNESCAPE);
    Py_INCREF(FIND_NEWLINES);
    Py_INCREF(TOKEN_DISPATCH);
    Py_INCREF(COLON);
    Py_INCREF(SCOLON);
    Py_INCREF(LPAR);
    Py_INCREF(RPAR);
    Py_INCREF(LBRACE);
    Py_INCREF(RBRACE);
    Py_INCREF(LBOX);
    Py_INCREF(RBOX);
    Py_INCREF(DELIM_TOK);
    Py_INCREF(INTEGER);
    Py_INCREF(STRING_TOK);

#define SETCONST(x)                                                                   \
    do {                                                                              \
        (x) = PyNumber_AsSsize_t(PyDict_GetItemString(cti, #x), PyExc_OverflowError); \
        if ((x) == -1 && PyErr_Occurred() != NULL) { return NULL; }                   \
    } while (0)
    SETCONST(BAD_COMMENT);
    SETCONST(BAD_STRING);
    SETCONST(PERCENTAGE);
    SETCONST(DIMENSION);
    SETCONST(ATKEYWORD);
    SETCONST(FUNCTION);
    SETCONST(COMMENT);
    SETCONST(NUMBER);
    SETCONST(STRING);
    SETCONST(IDENT);
    SETCONST(HASH);
    SETCONST(URI);

    Py_RETURN_NONE;
}

#if PY_VERSION_HEX >= 0x03030000
#define ITER_CODE_PTS(unicode_object)                                                             \
    {                                                                                             \
        int _kind = PyUnicode_KIND(unicode_object);                                               \
        void *_data = PyUnicode_DATA(unicode_object);                                             \
        for (Py_ssize_t iteridx = 0; iteridx < PyUnicode_GET_LENGTH(unicode_object); iteridx++) { \
            Py_UCS4 ch = PyUnicode_READ(_kind, _data, iteridx);
#else
#define PyUnicode_GET_LENGTH PyUnicode_GET_SIZE
#define ITER_CODE_PTS(unicode_object)                                                  \
    {                                                                                  \
        Py_UNICODE *_data = PyUnicode_AS_UNICODE(unicode_object);                      \
        Py_ssize_t iteridx;                                                            \
        for (iteridx = 0; iteridx < PyUnicode_GET_LENGTH(unicode_object); iteridx++) { \
            Py_UNICODE ch = _data[iteridx];
#endif

#define END_ITER_CODE_PTS \
    }                     \
    }

static PyObject *
unicode_to_number(PyObject *src) {
    PyObject *ans = PyFloat_FromString(src);
    double val = PyFloat_AsDouble(ans);
    long lval = (long)val;
    if (val - lval != 0) return ans;
    Py_DECREF(ans);
    return PyLong_FromLong(lval);
}


static void
lowercase(PyObject *x) {
    ITER_CODE_PTS(x)
    if ('A' <= ch && ch <= 'Z') {
#if PY_VERSION_HEX >= 0x03030000
        PyUnicode_WRITE(_kind, _data, iteridx, ch + 32);
#else
        _data[iteridx] += 32;
#endif
    }
    END_ITER_CODE_PTS
}

static PyObject *
clone_unicode(const PyObject *src, Py_ssize_t start_offset, Py_ssize_t end_offset) {
#if PY_VERSION_HEX >= 0x03030000
    int kind = PyUnicode_KIND(src);
    void *data;
    switch (kind) {
        case PyUnicode_1BYTE_KIND: data = PyUnicode_1BYTE_DATA(src) + start_offset; break;
        case PyUnicode_2BYTE_KIND: data = PyUnicode_2BYTE_DATA(src) + start_offset; break;
        case PyUnicode_4BYTE_KIND: data = PyUnicode_4BYTE_DATA(src) + start_offset; break;
        default: PyErr_SetString(PyExc_RuntimeError, "Invalid byte kind for unicode object"); return NULL;
    }
    return PyUnicode_FromKindAndData(kind, data, PyUnicode_GET_LENGTH(src) - start_offset - end_offset);
#else
    return PyUnicode_FromUnicode(PyUnicode_AS_UNICODE(src) + start_offset, PyUnicode_GET_LENGTH(src) - start_offset - end_offset);
#endif
}

static PyObject *
tokenize_flat(PyObject *self, PyObject *args) {
#if PY_VERSION_HEX >= 0x03030000
    void *css_source = NULL;
    int css_kind;
    Py_UCS4 c = 0, codepoint = 0;
#define first_char(string) PyUnicode_READ_CHAR(string, 0)
#define unicode_from_data(data, sz) PyUnicode_FromKindAndData(css_kind, data, sz)
#else
    Py_UNICODE *css_source = NULL, c = 0, codepoint = 0;
#define first_char(string) PyUnicode_AS_UNICODE(string)[0]
#define unicode_from_data(data, sz) PyUnicode_FromUnicode(data, sz)
#endif
    PyObject *ic = NULL, *token = NULL, *tokens = NULL, *type_name = NULL, *css_value = NULL, *value = NULL, *unit = NULL, *tries = NULL, *match = NULL,
             *match_func = NULL, *py_source = NULL, *item = NULL, *newlines = NULL;
    int ignore_comments = 0;
    Py_ssize_t pos = 0, line = 1, column = 1, i = 0;
    Py_ssize_t length = 0, next_pos = 0, type_ = -1, source_len = 0;


    if (COMPILED_TOKEN_REGEXPS == NULL) {
        PyErr_SetString(PyExc_RuntimeError, "tokenizer module not initialized. You must call init() first.");
        return NULL;
    }

    if (!PyArg_ParseTuple(args, "UO", &py_source, &ic)) return NULL;
    if (PyObject_IsTrue(ic)) ignore_comments = 1;
#if PY_VERSION_HEX >= 0x03030000
    if (PyUnicode_READY(py_source) != 0) return NULL;
    css_source = PyUnicode_DATA(py_source);
    css_kind = PyUnicode_KIND(py_source);
#else
    css_source = PyUnicode_AS_UNICODE(py_source);
#endif
    source_len = PyUnicode_GET_LENGTH(py_source);

    tokens = PyList_New(0);
    if (tokens == NULL) return PyErr_NoMemory();

#define UNESCAPE(x, func)                               \
    item = PyObject_CallFunctionObjArgs(func, x, NULL); \
    if (item == NULL) { goto error; }                   \
    Py_DECREF(x);                                       \
    x = item;                                           \
    item = NULL;

#define TONUMBER(x)               \
    item = unicode_to_number(x);  \
    if (item == NULL) goto error; \
    Py_DECREF(x);                 \
    x = item;                     \
    item = NULL;

#define SINGLE(x)             \
    {                         \
        type_ = -1;           \
        type_name = x;        \
        Py_INCREF(type_name); \
        css_value = x;        \
        Py_INCREF(css_value); \
    }

    while (pos < source_len) {
#if PY_VERSION_HEX >= 0x03030000
        c = PyUnicode_READ(css_kind, css_source, pos);
#else
        c = css_source[pos];
#endif

        css_value = NULL;
        type_name = NULL;
        value = NULL;
        unit = NULL;
        match = NULL;

        if (c == ':') SINGLE(COLON)
        else if (c == ';') SINGLE(SCOLON)
        else if (c == '(') SINGLE(LPAR)
        else if (c == ')') SINGLE(RPAR)
        else if (c == '{') SINGLE(LBRACE)
        else if (c == '}') SINGLE(RBRACE)
        else if (c == '[') SINGLE(LBOX)
        else if (c == ']') SINGLE(RBOX)
        else {
            codepoint = (c > 160) ? 160 : c;
            tries = PyList_GET_ITEM(TOKEN_DISPATCH, codepoint);
            for (i = 0; i < PyList_Size(tries); i++) {
                item = PyList_GET_ITEM(tries, i);
                match_func = PyTuple_GET_ITEM(item, 2);
                match = PyObject_CallFunction(match_func, "On", py_source, pos);
                if (match == NULL) { goto error; }
                if (match != Py_None) {
                    css_value = PyObject_CallMethod(match, "group", NULL);
                    if (css_value == NULL) { goto error; }
                    type_ = PyNumber_AsSsize_t(PyTuple_GET_ITEM(item, 0), PyExc_OverflowError);
                    if (type_ == -1 && PyErr_Occurred() != NULL) { goto error; }
                    type_name = PyTuple_GET_ITEM(item, 1);
                    Py_INCREF(type_name);
                    break;
                }
            }
            if (css_value == NULL) { // No match
                type_ = DELIM;
                type_name = DELIM_TOK;
                Py_INCREF(type_name);
                css_value = unicode_from_data(&c, 1);
                if (css_value == NULL) { goto error; }
            }
        }

        length = PyUnicode_GET_LENGTH(css_value);
        next_pos = pos + length;

        // Now calculate the value and unit for this token (if any)
        if (!(ignore_comments && (type_ == COMMENT || type_ == BAD_COMMENT))) {
            if (type_ == DIMENSION) {
                value = PyObject_CallMethod(match, "group", "I", 1);
                if (value == NULL) { goto error; }
                TONUMBER(value);
                unit = PyObject_CallMethod(match, "group", "I", 2);
                if (unit == NULL) { goto error; }
                UNESCAPE(unit, SIMPLE_UNESCAPE);
                UNESCAPE(unit, UNICODE_UNESCAPE);
                lowercase(unit);
            } else

                if (type_ == PERCENTAGE) {
                if (PyUnicode_GET_LENGTH(css_value) > 0) {
                    value = clone_unicode(css_value, 0, 1);
                    if (value == NULL) goto error;
                } else {
                    value = css_value;
                    Py_INCREF(value);
                }
                if (value == NULL) goto error;
                TONUMBER(value);
                unit = PyUnicode_FromString("%");
                if (unit == NULL) goto error;
            } else

                if (type_ == NUMBER) {
                value = css_value;
                Py_INCREF(value);
                TONUMBER(value);
                if (!PyFloat_Check(value)) {
                    Py_XDECREF(type_name);
                    type_name = INTEGER;
                    Py_INCREF(type_name);
                }
            } else

                if (type_ == IDENT || type_ == ATKEYWORD || type_ == HASH || type_ == FUNCTION) {
                value = PyObject_CallFunctionObjArgs(SIMPLE_UNESCAPE, css_value, NULL);
                if (value == NULL) goto error;
                UNESCAPE(value, UNICODE_UNESCAPE);
            } else

                if (type_ == URI) {
                value = PyObject_CallMethod(match, "group", "I", 1);
                if (value == NULL) { goto error; }
                if (PyObject_IsTrue(value) && PyUnicode_GET_LENGTH(value) > 1 && (first_char(value) == '"' || first_char(value) == '\'')) {
                    item = clone_unicode(value, 1, 1);
                    if (item == NULL) goto error;
                    Py_DECREF(value);
                    value = item;
                    item = NULL;
                    UNESCAPE(value, NEWLINE_UNESCAPE);
                }
                UNESCAPE(value, SIMPLE_UNESCAPE);
                UNESCAPE(value, UNICODE_UNESCAPE);
            } else

                if (type_ == STRING) {
                if (PyObject_IsTrue(css_value) && PyUnicode_GET_LENGTH(css_value) > 1) { // remove quotes
                    value = clone_unicode(css_value, 1, 1);
                } else {
                    value = css_value;
                    Py_INCREF(value);
                }
                UNESCAPE(value, NEWLINE_UNESCAPE);
                UNESCAPE(value, SIMPLE_UNESCAPE);
                UNESCAPE(value, UNICODE_UNESCAPE);
            } else

                if (type_ == BAD_STRING && next_pos == source_len) {
                Py_XDECREF(type_name);
                type_name = STRING_TOK;
                Py_INCREF(type_name);
                if (PyObject_IsTrue(css_value) && PyUnicode_GET_LENGTH(css_value) > 0) { // remove quote
                    value = clone_unicode(css_value, 1, 0);
                } else {
                    value = css_value;
                    Py_INCREF(value);
                }
                UNESCAPE(value, NEWLINE_UNESCAPE);
                UNESCAPE(value, SIMPLE_UNESCAPE);
                UNESCAPE(value, UNICODE_UNESCAPE);
            } else {
                value = css_value;
                Py_INCREF(value);
            } // if(type_ == ...)

            if (unit == NULL) {
                unit = Py_None;
                Py_INCREF(unit);
            }
            item = Py_BuildValue("OOOOnn", type_name, css_value, value, unit, line, column);
            if (item == NULL) goto error;
            token = PyObject_CallObject((PyObject *)&tokenizer_TokenType, item);
            Py_DECREF(item);
            item = NULL;
            if (token == NULL) goto error;
            if (PyList_Append(tokens, token) != 0) {
                Py_DECREF(token);
                token = NULL;
                goto error;
            }
            Py_DECREF(token);

        } // if(!(ignore_comments...

        Py_XDECREF(match);
        match = NULL;

        pos = next_pos;
        newlines = PyObject_CallFunctionObjArgs(FIND_NEWLINES, css_value, NULL);
        if (newlines == NULL) goto error;
        Py_XDECREF(css_value);
        css_value = NULL;
        Py_XDECREF(type_name);
        type_name = NULL;
        Py_XDECREF(value);
        value = NULL;
        Py_XDECREF(unit);
        unit = NULL;
        if (PyObject_IsTrue(newlines)) {
            line += PyList_Size(newlines);
            item = PyObject_CallMethod(PyList_GET_ITEM(newlines, PyList_Size(newlines) - 1), "end", NULL);
            if (item == NULL) {
                Py_DECREF(newlines);
                newlines = NULL;
                goto error;
            }
            column = PyNumber_AsSsize_t(item, PyExc_OverflowError);
            if (column == -1 && PyErr_Occurred()) {
                Py_DECREF(newlines);
                newlines = NULL;
                goto error;
            }
            column = length - column + 1;
            Py_DECREF(item);
            item = NULL;
        } else column += length;
        Py_DECREF(newlines);
        newlines = NULL;

    } // while (pos < ...)

    return tokens;
error:
    Py_XDECREF(tokens);
    Py_XDECREF(css_value);
    Py_XDECREF(type_name);
    Py_XDECREF(value);
    Py_XDECREF(unit);
    Py_XDECREF(match);
    return NULL;
#undef unicode_from_data
#undef first_char
}

static PyMethodDef tokenizer_methods[] = {
    {"tokenize_flat", tokenize_flat, METH_VARARGS, "tokenize_flat(css_source, ignore_comments)\n\n Convert CSS source into a flat list of tokens"},

    {"init", tokenize_init, METH_VARARGS, "init()\n\nInitialize the module."},

    {"cleanup", tokenize_cleanup, METH_VARARGS, "cleanup()\n\nRelease resources allocated by init(). Safe to call multiple times."},

    {NULL, NULL, 0, NULL}};

static int
exec_module(PyObject *mod) {
    if (PyType_Ready(&tokenizer_TokenType) < 0) return -1;
    Py_INCREF(&tokenizer_TokenType);
    PyModule_AddObject(mod, "Token", (PyObject *)&tokenizer_TokenType);
    return 0;
}
static PyModuleDef_Slot slots[] = {{Py_mod_exec, exec_module}, {0, NULL}};

static struct PyModuleDef module_def = {
    .m_base = PyModuleDef_HEAD_INIT,
    .m_name = "tokenizer",
    .m_doc = "Implementation of tokenizer in C for speed.",
    .m_methods = tokenizer_methods,
    .m_slots = slots,
};

CALIBRE_MODINIT_FUNC
PyInit_tokenizer(void) { return PyModuleDef_Init(&module_def); }
