diff --git a/amazon/ion/ioncmodule.c b/amazon/ion/ioncmodule.c index 99bb58dd4..3eda45527 100644 --- a/amazon/ion/ioncmodule.c +++ b/amazon/ion/ioncmodule.c @@ -1527,15 +1527,36 @@ void ionc_read_iter_dealloc(PyObject *self) { PyObject_Del(self); } +// TODO Convert C symbol table information back to Python layer +iERR c_symbol_table_to_py_symbol_table(hREADER hreader ) { + iENTER; + printf("converting..."); + iRETURN; +} + +// TODO Convert Python symbol table to a C symbol table +iERR python_symbol_table_to_py_symbol_table(hREADER hreader ) { + iENTER; + printf("converting..."); + iRETURN; +} + + + iERR ionc_lazy_read_all(hREADER hreader, PyObject* container, BOOL in_struct, BOOL emit_bare_values, char* buffer) { iENTER; ION_TYPE t; + BOOL cached_symtbl = TRUE; + PyObject* py_cached_sym_table_bytes = Py_None; + for (;;) { + // Do it first time and cache symbol table bytes IONCHECK(ion_reader_next(hreader, &t)); if (t == tid_EOF) { assert(t == tid_EOF && "next() at end"); break; } + PyObject* rtn; PyObject* py_cached_bytes; @@ -1550,6 +1571,11 @@ iERR ionc_lazy_read_all(hREADER hreader, PyObject* container, BOOL in_struct, BO printf("p_offset is: %" PRId64 "\n", p_offset); printf("p_length is: %" PRId32 "\n\n", p_length); + if (cached_symtbl) { + py_cached_sym_table_bytes = Py_BuildValue("y#", buffer+4, p_offset-4); + cached_symtbl = FALSE; + } + // TODO is it possible to return a memory view pointing to the specific position of the original buffer? // Python Bytes? Py_BuildValue("y#", buffer+p_offset, p_length), // Python MemoryView? PyMemoryView_FromMemory(buffer+p_offset, p_length, PyBUF_WRITE), @@ -1561,6 +1587,7 @@ iERR ionc_lazy_read_all(hREADER hreader, PyObject* container, BOOL in_struct, BO _ionpylazyobj_cls, py_cached_bytes, py_ion_type_table[ION_TYPE_INT(t) >> 8], + py_cached_sym_table_bytes, NULL ); @@ -1587,6 +1614,7 @@ PyObject* ionc_read(PyObject* self, PyObject *args, PyObject *kwds) { // Store the stream in IonPyObj until it actually needs to be serialized. if (parse_lazily == Py_True) { hREADER reader; + hSYMTAB sym_table; char *buffer = NULL; long size; PyObject *top_level_container = NULL; @@ -1602,6 +1630,54 @@ PyObject* ionc_read(PyObject* self, PyObject *args, PyObject *kwds) { top_level_container = PyList_New(0); IONCHECK(ionc_lazy_read_all(reader, top_level_container, FALSE, emit_bare_values == Py_True, buffer)); + +// IONCHECK(ion_reader_get_symbol_table(reader, &sym_table)); +// int32_t v; +// ION_STRING n; +// SID id; +// BOOL know; +// IONCHECK(ion_symbol_table_get_version(sym_table, &v)); +// IONCHECK(ion_symbol_table_get_name(sym_table, &n)); +// IONCHECK(ion_symbol_table_get_max_sid(sym_table, &id)); +// printf("version is: %" PRId32 "\n", v); +// printf("name is: %s\n", n.value); +// printf("max id is: %" PRId32 "\n", id); +// +// ION_SYMBOL_TABLE_TYPE t; +// IONCHECK(ion_symbol_table_get_type(sym_table, &t)); +// printf("symbol table type is: %d\n", t); +// +// hWRITER writer; +// ION_WRITER_OPTIONS options2; +// ION_STREAM *ion_stream = NULL; +// IONCHECK(ion_stream_open_memory_only(&ion_stream)); +// ION_STRING is; +// char* cs = "ahj"; +// ion_string_assign_cstr(&is, cs, 3); +// char* buf; +// +// memset(&options2, 0, sizeof(options2)); +// options2.output_as_binary = TRUE; +// options2.max_annotation_count = ANNOTATION_MAX_LEN; +// IONCHECK(ion_writer_open(&writer, ion_stream, &options2)); +// +// IONCHECK(ion_writer_set_symbol_table(writer, sym_table)); +// IONCHECK(ion_writer_write_symbol(writer, &is)); +// +// IONCHECK(ion_writer_close(writer)); +// writer = 0; +// +// POSITION len = ion_stream_get_position(ion_stream); +// IONCHECK(ion_stream_seek(ion_stream, 0)); +// buf = (BYTE*)(PyMem_Malloc((size_t)len)); +// SIZE bytes_read; +// IONCHECK(ion_stream_read(ion_stream, buf, (SIZE)len, &bytes_read)); +// IONCHECK(ion_stream_close(ion_stream)); +// +// PyObject* written = Py_BuildValue(IONC_BYTES_FORMAT, (char*)buf, bytes_read); +// return written; + + IONCHECK(ion_reader_close(reader)); return top_level_container; } else { diff --git a/amazon/ion/lazy_type.py b/amazon/ion/lazy_type.py index d7a05846a..36b790fb9 100644 --- a/amazon/ion/lazy_type.py +++ b/amazon/ion/lazy_type.py @@ -9,11 +9,13 @@ class IonPyLazyObj(_IonNature): """ ion_buffer = None ion_type = None + symbol_table = None - def __init__(self, b, t, *args, **kwargs): + def __init__(self, b, t, s, *args, **kwargs): super().__init__(*args, **kwargs) self.lazy_buffer = b self.lazy_type = t + self.symbol_table = s # Wake up the lazy object, return a real IonPyObj # This might be helpful later, but is not used at all for now diff --git a/amazon/ion/test.py b/amazon/ion/test.py index bb6d1139d..e602c4346 100644 --- a/amazon/ion/test.py +++ b/amazon/ion/test.py @@ -17,3 +17,15 @@ # Text representation IVM [ blob([ 1, 2 ]) blob([ 3 ])] # We should take out the blob wrapper later. print(ion.dumps(obj)) + + + +ion_binary_bytes_dict = b'\xe0\x01\x00\xea\xe7\x81\x83\xd4\x87\xb2\x81a\xd3\x8a!\x02' +obj = ion.loads(ion_binary_bytes_dict, parse_lazily=True) + + + + + + + diff --git a/amazon/ion/test_for_symbol.py b/amazon/ion/test_for_symbol.py new file mode 100644 index 000000000..ca2cd20c8 --- /dev/null +++ b/amazon/ion/test_for_symbol.py @@ -0,0 +1,16 @@ +import amazon.ion.simpleion as ion + +# {a: 2} +ion_symbol_table_bytes = b'\xe0\x01\x00\xea \xe8\x81\x83\xde\x84\x87\xb2\x81a \xde\x83\x8a\x21\x02' +print(ion_symbol_table_bytes) +obj = ion.loads(ion_symbol_table_bytes, parse_lazily=True) + +# This returns '\xe8\x81\x83\xde\x84\x87\xb2\x81a' part. +print(obj[0].symbol_table) + + + + + + +