ijpython.py

'''
Pure-python parsing backend.
'''
from __future__ import unicode_literals
import decimal
import re

from ijson import common


# BUFSIZE = 16 * 1024
BUFSIZE = 4 * 1024
LEXEME_RE = re.compile(r'[a-z0-9eE\.\+-]+|\S')
# my_lexer = None


class UnexpectedSymbol(common.JSONError):
    def __init__(self, symbol, pos):
        super(UnexpectedSymbol, self).__init__(
            'Unexpected symbol %r at %d' % (symbol, pos)
        )


def Lexer(f, buf_size=BUFSIZE):
    buf = f.read(buf_size)
    pos = 0
    discarded = 0
    while True:
        match = LEXEME_RE.search(buf, pos)
        if match:
            lexeme = match.group()
            if lexeme == '"':
                pos = match.start()
                start = pos + 1
                while True:
                    try:
                        end = buf.index('"', start)
                        escpos = end - 1
                        while buf[escpos] == '\\':
                            escpos -= 1
                        if (end - escpos) % 2 == 0:
                            start = end + 1
                        else:
                            break
                    except ValueError:
                        data = f.read(buf_size)
                        if not data:
                            print buf, pos, end
                            print buf[pos:end + 1]
                            raise common.IncompleteJSONError('Incomplete string lexeme')
                        buf += data
                yield discarded + pos, buf[pos:end + 1]
                pos = end + 1
            else:
                while match.end() == len(buf):
                    data = f.read(buf_size)
                    if not data:
                        break
                    buf += data
                    match = LEXEME_RE.search(buf, pos)
                    lexeme = match.group()
                yield discarded + match.start(), lexeme
                pos = match.end()
        else:
            data = f.read(buf_size)
            if not data:
                break
            discarded += len(buf)
            buf = data
            pos = 0


def unescape(s):
    start = 0
    result = ''
    while start < len(s):
        pos = s.find('\\', start)
        if pos == -1:
            if start == 0:
                return s
            result += s[start:]
            break
        result += s[start:pos]
        pos += 1
        esc = s[pos]
        if esc == 'u':
            result += unichr(int(s[pos + 1:pos + 5], 16))
            pos += 4
        elif esc == 'b':
            result += '\b'
        elif esc == 'f':
            result += '\f'
        elif esc == 'n':
            result += '\n'
        elif esc == 'r':
            result += '\r'
        elif esc == 't':
            result += '\t'
        else:
            result += esc
        start = pos + 1
    return result


def parse_value(lexer, symbol=None, pos=0):
    try:
        if symbol is None:
            pos, symbol = next(lexer)
        if symbol == 'null':
            yield ('null', None)
        elif symbol == 'true':
            yield ('boolean', True)
        elif symbol == 'false':
            yield ('boolean', False)
        elif symbol == '[':
            for event in parse_array(lexer):
                yield event
        elif symbol == '{':
            for event in parse_object(lexer):
                yield event
        elif symbol[0] == '"':
            yield ('string', unescape(symbol[1:-1]))
        else:
            try:
                yield ('number', common.number(symbol))
            except decimal.InvalidOperation:
                raise UnexpectedSymbol(symbol, pos)
    except StopIteration:
        raise common.IncompleteJSONError('Incomplete JSON data')


def parse_array(lexer):
    yield ('start_array', None)
    try:
        pos, symbol = next(lexer)
        if symbol != ']':
            while True:
                for event in parse_value(lexer, symbol, pos):
                    yield event
                pos, symbol = next(lexer)
                if symbol == ']':
                    break
                if symbol != ',':
                    raise UnexpectedSymbol(symbol, pos)
                pos, symbol = next(lexer)
        yield ('end_array', None)
    except StopIteration:
        raise common.IncompleteJSONError('Incomplete JSON data')


def parse_object(lexer):
    yield ('start_map', None)
    try:
        pos, symbol = next(lexer)
        if symbol != '}':
            while True:
                if symbol[0] != '"':
                    raise UnexpectedSymbol(symbol, pos)
                yield ('map_key', unescape(symbol[1:-1]))
                pos, symbol = next(lexer)
                if symbol != ':':
                    raise UnexpectedSymbol(symbol, pos)
                for event in parse_value(lexer, None, pos):
                    yield event
                pos, symbol = next(lexer)
                if symbol == '}':
                    break
                if symbol != ',':
                    raise UnexpectedSymbol(symbol, pos)
                pos, symbol = next(lexer)
        yield ('end_map', None)
    except StopIteration:
        raise common.IncompleteJSONError('Incomplete JSON data')


# def basic_parse(f, buf_size=BUFSIZE):
#     '''
#     Iterator yielding unprefixed events.
# 
#     Parameters:
# 
#     - file: a readable file-like object with JSON input
#     '''
#     try:
#         global my_lexer
#         if not my_lexer:
#             my_lexer = iter(Lexer(f, buf_size))
#         for value in parse_value(my_lexer):
#             yield value
#     except StopIteration:
#         pass
# 
# #     lexer = iter(Lexer(file, buf_size))
# #     for value in parse_value(lexer):
# #         yield value
# #     try:
# #         print next(lexer)
# #     except StopIteration:
# #         pass
# #     else:
# #         raise common.JSONError('Additional data')
# 
# 
# def parse(f, buf_size=BUFSIZE):
#     '''
#     Backend-specific wrapper for ijson.common.parse.
#     '''
#     return common.parse(basic_parse(f, buf_size=buf_size))
# 
# 
# def items(f, prefix = ''):
#     '''
#     Backend-specific wrapper for ijson.common.items.
#     '''
#     return common.items(parse(f), prefix)

class JsonItems(object):
    def __init__(self, f, buf_size = BUFSIZE):
        self._lexer = Lexer(f, buf_size)
        
    def parse(self):
        return common.parse(self.__basic_parse())
    
    def get_item(self, prefix = ''):
        return next(iter(common.items(self.parse(), prefix)))
    
    def __basic_parse(self):
        try:
            for value in parse_value(self._lexer):
                yield value
        except StopIteration:
            pass