diff --git a/Modules/_bz2module.c b/Modules/_bz2module.c index 914172684158a1..ea94f4321d74de 100644 --- a/Modules/_bz2module.c +++ b/Modules/_bz2module.c @@ -669,9 +669,7 @@ _bz2_BZ2Decompressor_impl(PyTypeObject *type) self->bzs_avail_in_real = 0; self->input_buffer = NULL; self->input_buffer_size = 0; - self->unused_data = PyBytes_FromStringAndSize(NULL, 0); - if (self->unused_data == NULL) - goto error; + self->unused_data = Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); bzerror = BZ2_bzDecompressInit(&self->bzs, 0, 0); if (catch_bz2_error(bzerror)) diff --git a/Modules/_dbmmodule.c b/Modules/_dbmmodule.c index 17aca2f00a13c0..3fdcf22ffd56d0 100644 --- a/Modules/_dbmmodule.c +++ b/Modules/_dbmmodule.c @@ -450,10 +450,7 @@ _dbm_dbm_setdefault_impl(dbmobject *self, PyTypeObject *cls, const char *key, return PyBytes_FromStringAndSize(val.dptr, val.dsize); } if (default_value == NULL) { - default_value = PyBytes_FromStringAndSize(NULL, 0); - if (default_value == NULL) { - return NULL; - } + default_value = Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); val.dptr = NULL; val.dsize = 0; } diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c index 0b0b1bc765bbc9..bac49b1268cc56 100644 --- a/Modules/_lzmamodule.c +++ b/Modules/_lzmamodule.c @@ -1263,10 +1263,7 @@ _lzma_LZMADecompressor_impl(PyTypeObject *type, int format, self->needs_input = 1; self->input_buffer = NULL; self->input_buffer_size = 0; - Py_XSETREF(self->unused_data, PyBytes_FromStringAndSize(NULL, 0)); - if (self->unused_data == NULL) { - goto error; - } + Py_XSETREF(self->unused_data, Py_GetConstant(Py_CONSTANT_EMPTY_BYTES)); switch (format) { case FORMAT_AUTO: @@ -1445,7 +1442,7 @@ _lzma__encode_filter_properties_impl(PyObject *module, lzma_filter filter) { lzma_ret lzret; uint32_t encoded_size; - PyObject *result = NULL; + PyBytesWriter *writer = NULL; _lzma_state *state = get_lzma_state(module); assert(state != NULL); @@ -1453,20 +1450,20 @@ _lzma__encode_filter_properties_impl(PyObject *module, lzma_filter filter) if (catch_lzma_error(state, lzret)) goto error; - result = PyBytes_FromStringAndSize(NULL, encoded_size); - if (result == NULL) + writer = PyBytesWriter_Create(encoded_size); + if (writer == NULL) { goto error; + } - lzret = lzma_properties_encode( - &filter, (uint8_t *)PyBytes_AS_STRING(result)); + lzret = lzma_properties_encode(&filter, PyBytesWriter_GetData(writer)); if (catch_lzma_error(state, lzret)) { goto error; } - return result; + return PyBytesWriter_Finish(writer); error: - Py_XDECREF(result); + PyBytesWriter_Discard(writer); return NULL; } diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 0774e91039ec41..bfb2830f3893d6 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -2612,31 +2612,26 @@ save_picklebuffer(PickleState *st, PicklerObject *self, PyObject *obj) static PyObject * raw_unicode_escape(PyObject *obj) { - char *p; - Py_ssize_t i, size; - const void *data; - int kind; - _PyBytesWriter writer; + Py_ssize_t size = PyUnicode_GET_LENGTH(obj); + const void *data = PyUnicode_DATA(obj); + int kind = PyUnicode_KIND(obj); - _PyBytesWriter_Init(&writer); - - size = PyUnicode_GET_LENGTH(obj); - data = PyUnicode_DATA(obj); - kind = PyUnicode_KIND(obj); - - p = _PyBytesWriter_Alloc(&writer, size); - if (p == NULL) - goto error; - writer.overallocate = 1; + Py_ssize_t alloc = size; + PyBytesWriter *writer = PyBytesWriter_Create(alloc); + if (writer == NULL) { + return NULL; + } + char *p = PyBytesWriter_GetData(writer); - for (i=0; i < size; i++) { + for (Py_ssize_t i=0; i < size; i++) { Py_UCS4 ch = PyUnicode_READ(kind, data, i); /* Map 32-bit characters to '\Uxxxxxxxx' */ if (ch >= 0x10000) { /* -1: subtract 1 preallocated byte */ - p = _PyBytesWriter_Prepare(&writer, p, 10-1); - if (p == NULL) + p = PyBytesWriter_GrowAndUpdatePointer(writer, 10-1, p); + if (p == NULL) { goto error; + } *p++ = '\\'; *p++ = 'U'; @@ -2655,9 +2650,10 @@ raw_unicode_escape(PyObject *obj) ch == 0x1a) { /* -1: subtract 1 preallocated byte */ - p = _PyBytesWriter_Prepare(&writer, p, 6-1); - if (p == NULL) + p = PyBytesWriter_GrowAndUpdatePointer(writer, 6-1, p); + if (p == NULL) { goto error; + } *p++ = '\\'; *p++ = 'u'; @@ -2671,10 +2667,10 @@ raw_unicode_escape(PyObject *obj) *p++ = (char) ch; } - return _PyBytesWriter_Finish(&writer, p); + return PyBytesWriter_FinishWithPointer(writer, p); error: - _PyBytesWriter_Dealloc(&writer); + PyBytesWriter_Discard(writer); return NULL; } diff --git a/Modules/_struct.c b/Modules/_struct.c index 3fad35a8c94ee2..f09252e82c3915 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -2189,7 +2189,6 @@ strings."); static PyObject * s_pack(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { - char *buf; PyStructObject *soself; _structmodulestate *state = get_struct_state_structinst(self); @@ -2205,21 +2204,19 @@ s_pack(PyObject *self, PyObject *const *args, Py_ssize_t nargs) } /* Allocate a new string */ - _PyBytesWriter writer; - _PyBytesWriter_Init(&writer); - buf = _PyBytesWriter_Alloc(&writer, soself->s_size); - if (buf == NULL) { - _PyBytesWriter_Dealloc(&writer); + PyBytesWriter *writer = PyBytesWriter_Create(soself->s_size); + if (writer == NULL) { return NULL; } + char *buf = PyBytesWriter_GetData(writer); /* Call the guts */ if ( s_pack_internal(soself, args, 0, buf, state) != 0 ) { - _PyBytesWriter_Dealloc(&writer); + PyBytesWriter_Discard(writer); return NULL; } - return _PyBytesWriter_Finish(&writer, buf + soself->s_size); + return PyBytesWriter_FinishWithSize(writer, soself->s_size); } PyDoc_STRVAR(s_pack_into__doc__, diff --git a/Modules/binascii.c b/Modules/binascii.c index c70f9f88afe14d..13e4bc5be03ebd 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -205,11 +205,9 @@ binascii_a2b_uu_impl(PyObject *module, Py_buffer *data) /*[clinic end generated code: output=e027f8e0b0598742 input=7cafeaf73df63d1c]*/ { const unsigned char *ascii_data; - unsigned char *bin_data; int leftbits = 0; unsigned char this_ch; unsigned int leftchar = 0; - PyObject *rv; Py_ssize_t ascii_len, bin_len; binascii_state *state; @@ -223,9 +221,11 @@ binascii_a2b_uu_impl(PyObject *module, Py_buffer *data) ascii_len--; /* Allocate the buffer */ - if ( (rv=PyBytes_FromStringAndSize(NULL, bin_len)) == NULL ) + PyBytesWriter *writer = PyBytesWriter_Create(bin_len); + if (writer == NULL) { return NULL; - bin_data = (unsigned char *)PyBytes_AS_STRING(rv); + } + unsigned char *bin_data = PyBytesWriter_GetData(writer); for( ; bin_len > 0 ; ascii_len--, ascii_data++ ) { /* XXX is it really best to add NULs if there's no more data */ @@ -245,11 +245,10 @@ binascii_a2b_uu_impl(PyObject *module, Py_buffer *data) if ( this_ch < ' ' || this_ch > (' ' + 64)) { state = get_binascii_state(module); if (state == NULL) { - return NULL; + goto error; } PyErr_SetString(state->Error, "Illegal char"); - Py_DECREF(rv); - return NULL; + goto error; } this_ch = (this_ch - ' ') & 077; } @@ -277,14 +276,17 @@ binascii_a2b_uu_impl(PyObject *module, Py_buffer *data) this_ch != '\n' && this_ch != '\r' ) { state = get_binascii_state(module); if (state == NULL) { - return NULL; + goto error; } PyErr_SetString(state->Error, "Trailing garbage"); - Py_DECREF(rv); - return NULL; + goto error; } } - return rv; + return PyBytesWriter_Finish(writer); + +error: + PyBytesWriter_Discard(writer); + return NULL; } /*[clinic input] @@ -302,16 +304,13 @@ static PyObject * binascii_b2a_uu_impl(PyObject *module, Py_buffer *data, int backtick) /*[clinic end generated code: output=b1b99de62d9bbeb8 input=beb27822241095cd]*/ { - unsigned char *ascii_data; const unsigned char *bin_data; int leftbits = 0; unsigned char this_ch; unsigned int leftchar = 0; binascii_state *state; - Py_ssize_t bin_len, out_len; - _PyBytesWriter writer; + Py_ssize_t bin_len; - _PyBytesWriter_Init(&writer); bin_data = data->buf; bin_len = data->len; if ( bin_len > 45 ) { @@ -325,10 +324,12 @@ binascii_b2a_uu_impl(PyObject *module, Py_buffer *data, int backtick) } /* We're lazy and allocate to much (fixed up later) */ - out_len = 2 + (bin_len + 2) / 3 * 4; - ascii_data = _PyBytesWriter_Alloc(&writer, out_len); - if (ascii_data == NULL) + Py_ssize_t out_len = 2 + (bin_len + 2) / 3 * 4; + PyBytesWriter *writer = PyBytesWriter_Create(out_len); + if (writer == NULL) { return NULL; + } + unsigned char *ascii_data = PyBytesWriter_GetData(writer); /* Store the length */ if (backtick && !bin_len) @@ -356,7 +357,7 @@ binascii_b2a_uu_impl(PyObject *module, Py_buffer *data, int backtick) } *ascii_data++ = '\n'; /* Append a courtesy newline */ - return _PyBytesWriter_Finish(&writer, ascii_data); + return PyBytesWriter_FinishWithPointer(writer, ascii_data); } /*[clinic input] @@ -388,12 +389,11 @@ binascii_a2b_base64_impl(PyObject *module, Py_buffer *data, int strict_mode) /* Allocate the buffer */ Py_ssize_t bin_len = ((ascii_len+3)/4)*3; /* Upper bound, corrected later */ - _PyBytesWriter writer; - _PyBytesWriter_Init(&writer); - unsigned char *bin_data = _PyBytesWriter_Alloc(&writer, bin_len); - if (bin_data == NULL) + PyBytesWriter *writer = PyBytesWriter_Create(bin_len); + if (writer == NULL) { return NULL; - unsigned char *bin_data_start = bin_data; + } + unsigned char *bin_data = PyBytesWriter_GetData(writer); if (strict_mode && ascii_len > 0 && ascii_data[0] == '=') { state = get_binascii_state(module); @@ -489,12 +489,14 @@ binascii_a2b_base64_impl(PyObject *module, Py_buffer *data, int strict_mode) state = get_binascii_state(module); if (state == NULL) { /* error already set, from get_binascii_state */ + assert(PyErr_Occurred()); } else if (quad_pos == 1) { /* ** There is exactly one extra valid, non-padding, base64 character. ** This is an invalid length, as there is no possible input that ** could encoded into such a base64 string. */ + unsigned char *bin_data_start = PyBytesWriter_GetData(writer); PyErr_Format(state->Error, "Invalid base64-encoded string: " "number of data characters (%zd) cannot be 1 more " @@ -503,13 +505,15 @@ binascii_a2b_base64_impl(PyObject *module, Py_buffer *data, int strict_mode) } else { PyErr_SetString(state->Error, "Incorrect padding"); } - error_end: - _PyBytesWriter_Dealloc(&writer); - return NULL; + goto error_end; } done: - return _PyBytesWriter_Finish(&writer, bin_data); + return PyBytesWriter_FinishWithPointer(writer, bin_data); + +error_end: + PyBytesWriter_Discard(writer); + return NULL; } @@ -528,18 +532,15 @@ static PyObject * binascii_b2a_base64_impl(PyObject *module, Py_buffer *data, int newline) /*[clinic end generated code: output=4ad62c8e8485d3b3 input=0e20ff59c5f2e3e1]*/ { - unsigned char *ascii_data; const unsigned char *bin_data; int leftbits = 0; unsigned char this_ch; unsigned int leftchar = 0; - Py_ssize_t bin_len, out_len; - _PyBytesWriter writer; + Py_ssize_t bin_len; binascii_state *state; bin_data = data->buf; bin_len = data->len; - _PyBytesWriter_Init(&writer); assert(bin_len >= 0); @@ -555,12 +556,15 @@ binascii_b2a_base64_impl(PyObject *module, Py_buffer *data, int newline) /* We're lazy and allocate too much (fixed up later). "+2" leaves room for up to two pad characters. Note that 'b' gets encoded as 'Yg==\n' (1 in, 5 out). */ - out_len = bin_len*2 + 2; - if (newline) + Py_ssize_t out_len = bin_len*2 + 2; + if (newline) { out_len++; - ascii_data = _PyBytesWriter_Alloc(&writer, out_len); - if (ascii_data == NULL) + } + PyBytesWriter *writer = PyBytesWriter_Create(out_len); + if (writer == NULL) { return NULL; + } + unsigned char *ascii_data = PyBytesWriter_GetData(writer); for( ; bin_len > 0 ; bin_len--, bin_data++ ) { /* Shift the data into our buffer */ @@ -585,7 +589,7 @@ binascii_b2a_base64_impl(PyObject *module, Py_buffer *data, int newline) if (newline) *ascii_data++ = '\n'; /* Append a courtesy newline */ - return _PyBytesWriter_Finish(&writer, ascii_data); + return PyBytesWriter_FinishWithPointer(writer, ascii_data); } @@ -887,8 +891,6 @@ binascii_a2b_hex_impl(PyObject *module, Py_buffer *hexstr) { const char* argbuf; Py_ssize_t arglen; - PyObject *retval; - char* retbuf; Py_ssize_t i, j; binascii_state *state; @@ -910,10 +912,11 @@ binascii_a2b_hex_impl(PyObject *module, Py_buffer *hexstr) return NULL; } - retval = PyBytes_FromStringAndSize(NULL, (arglen/2)); - if (!retval) + PyBytesWriter *writer = PyBytesWriter_Create(arglen/2); + if (writer == NULL) { return NULL; - retbuf = PyBytes_AS_STRING(retval); + } + char *retbuf = PyBytesWriter_GetData(writer); for (i=j=0; i < arglen; i += 2) { unsigned int top = _PyLong_DigitValue[Py_CHARMASK(argbuf[i])]; @@ -921,18 +924,18 @@ binascii_a2b_hex_impl(PyObject *module, Py_buffer *hexstr) if (top >= 16 || bot >= 16) { state = get_binascii_state(module); if (state == NULL) { - return NULL; + goto error; } PyErr_SetString(state->Error, "Non-hexadecimal digit found"); - goto finally; + goto error; } retbuf[j++] = (top << 4) + bot; } - return retval; + return PyBytesWriter_Finish(writer); - finally: - Py_DECREF(retval); +error: + PyBytesWriter_Discard(writer); return NULL; } diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 8c9d463ad903ec..8caadde8ae211b 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -448,7 +448,8 @@ _safe_PyBytes_ReverseFind(Py_ssize_t *out, mmap_object *self, } PyObject * -_safe_PyBytes_FromStringAndSize(char *start, size_t num_bytes) { +_safe_PyBytes_FromStringAndSize(char *start, size_t num_bytes) +{ if (num_bytes == 1) { char dest; if (safe_byte_copy(&dest, start) < 0) { @@ -459,14 +460,15 @@ _safe_PyBytes_FromStringAndSize(char *start, size_t num_bytes) { } } else { - PyObject *result = PyBytes_FromStringAndSize(NULL, num_bytes); - if (result == NULL) { + PyBytesWriter *writer = PyBytesWriter_Create(num_bytes); + if (writer == NULL) { return NULL; } - if (safe_memcpy(PyBytes_AS_STRING(result), start, num_bytes) < 0) { - Py_CLEAR(result); + if (safe_memcpy(PyBytesWriter_GetData(writer), start, num_bytes) < 0) { + PyBytesWriter_Discard(writer); + return NULL; } - return result; + return PyBytesWriter_Finish(writer); } } diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index f229c4e8dd9322..5e735e86bdee9e 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -11507,9 +11507,6 @@ static PyObject * os_read_impl(PyObject *module, int fd, Py_ssize_t length) /*[clinic end generated code: output=dafbe9a5cddb987b input=1df2eaa27c0bf1d3]*/ { - Py_ssize_t n; - PyObject *buffer; - if (length < 0) { errno = EINVAL; return posix_error(); @@ -11517,20 +11514,18 @@ os_read_impl(PyObject *module, int fd, Py_ssize_t length) length = Py_MIN(length, _PY_READ_MAX); - buffer = PyBytes_FromStringAndSize((char *)NULL, length); - if (buffer == NULL) + PyBytesWriter *writer = PyBytesWriter_Create(length); + if (writer == NULL) { return NULL; + } - n = _Py_read(fd, PyBytes_AS_STRING(buffer), length); + Py_ssize_t n = _Py_read(fd, PyBytesWriter_GetData(writer), length); if (n == -1) { - Py_DECREF(buffer); + PyBytesWriter_Discard(writer); return NULL; } - if (n != length) - _PyBytes_Resize(&buffer, n); - - return buffer; + return PyBytesWriter_FinishWithSize(writer, n); } /*[clinic input] @@ -11708,20 +11703,20 @@ os_pread_impl(PyObject *module, int fd, Py_ssize_t length, Py_off_t offset) { Py_ssize_t n; int async_err = 0; - PyObject *buffer; if (length < 0) { errno = EINVAL; return posix_error(); } - buffer = PyBytes_FromStringAndSize((char *)NULL, length); - if (buffer == NULL) + PyBytesWriter *writer = PyBytesWriter_Create(length); + if (writer == NULL) { return NULL; + } do { Py_BEGIN_ALLOW_THREADS _Py_BEGIN_SUPPRESS_IPH - n = pread(fd, PyBytes_AS_STRING(buffer), length, offset); + n = pread(fd, PyBytesWriter_GetData(writer), length, offset); _Py_END_SUPPRESS_IPH Py_END_ALLOW_THREADS } while (n < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); @@ -11730,12 +11725,10 @@ os_pread_impl(PyObject *module, int fd, Py_ssize_t length, Py_off_t offset) if (!async_err) { posix_error(); } - Py_DECREF(buffer); + PyBytesWriter_Discard(writer); return NULL; } - if (n != length) - _PyBytes_Resize(&buffer, n); - return buffer; + return PyBytesWriter_FinishWithSize(writer, n); } #endif /* HAVE_PREAD */ @@ -14945,9 +14938,6 @@ os_getxattr_impl(PyObject *module, path_t *path, path_t *attribute, int follow_symlinks) /*[clinic end generated code: output=5f2f44200a43cff2 input=025789491708f7eb]*/ { - Py_ssize_t i; - PyObject *buffer = NULL; - if (fd_and_follow_symlinks_invalid("getxattr", path->fd, follow_symlinks)) return NULL; @@ -14955,8 +14945,7 @@ os_getxattr_impl(PyObject *module, path_t *path, path_t *attribute, return NULL; } - for (i = 0; ; i++) { - void *ptr; + for (Py_ssize_t i = 0; ; i++) { ssize_t result; static const Py_ssize_t buffer_sizes[] = {128, XATTR_SIZE_MAX, 0}; Py_ssize_t buffer_size = buffer_sizes[i]; @@ -14964,10 +14953,11 @@ os_getxattr_impl(PyObject *module, path_t *path, path_t *attribute, path_error(path); return NULL; } - buffer = PyBytes_FromStringAndSize(NULL, buffer_size); - if (!buffer) + PyBytesWriter *writer = PyBytesWriter_Create(buffer_size); + if (writer == NULL) { return NULL; - ptr = PyBytes_AS_STRING(buffer); + } + void *ptr = PyBytesWriter_GetData(writer); Py_BEGIN_ALLOW_THREADS; if (path->fd >= 0) @@ -14979,23 +14969,16 @@ os_getxattr_impl(PyObject *module, path_t *path, path_t *attribute, Py_END_ALLOW_THREADS; if (result < 0) { + PyBytesWriter_Discard(writer); if (errno == ERANGE) { - Py_DECREF(buffer); continue; } path_error(path); - Py_DECREF(buffer); return NULL; } - if (result != buffer_size) { - /* Can only shrink. */ - _PyBytes_Resize(&buffer, result); - } - break; + return PyBytesWriter_FinishWithSize(writer, result); } - - return buffer; } @@ -15222,19 +15205,22 @@ static PyObject * os_urandom_impl(PyObject *module, Py_ssize_t size) /*[clinic end generated code: output=42c5cca9d18068e9 input=58a0def87dbc2c22]*/ { - PyObject *bytes; - int result; + if (size < 0) { + return PyErr_Format(PyExc_ValueError, + "negative argument not allowed"); + } - bytes = PyBytes_FromStringAndSize(NULL, size); - if (bytes == NULL) + PyBytesWriter *writer = PyBytesWriter_Create(size); + if (writer == NULL) { return NULL; + } - result = _PyOS_URandom(PyBytes_AS_STRING(bytes), PyBytes_GET_SIZE(bytes)); + int result = _PyOS_URandom(PyBytesWriter_GetData(writer), size); if (result == -1) { - Py_DECREF(bytes); + PyBytesWriter_Discard(writer); return NULL; } - return bytes; + return PyBytesWriter_Finish(writer); } #ifdef HAVE_MEMFD_CREATE @@ -16704,25 +16690,20 @@ static PyObject * os_getrandom_impl(PyObject *module, Py_ssize_t size, int flags) /*[clinic end generated code: output=b3a618196a61409c input=59bafac39c594947]*/ { - PyObject *bytes; - Py_ssize_t n; - if (size < 0) { errno = EINVAL; return posix_error(); } - bytes = PyBytes_FromStringAndSize(NULL, size); - if (bytes == NULL) { - PyErr_NoMemory(); + PyBytesWriter *writer = PyBytesWriter_Create(size); + if (writer == NULL) { return NULL; } + void *data = PyBytesWriter_GetData(writer); + Py_ssize_t n; while (1) { - n = syscall(SYS_getrandom, - PyBytes_AS_STRING(bytes), - PyBytes_GET_SIZE(bytes), - flags); + n = syscall(SYS_getrandom, data, size, flags); if (n < 0 && errno == EINTR) { if (PyErr_CheckSignals() < 0) { goto error; @@ -16739,14 +16720,10 @@ os_getrandom_impl(PyObject *module, Py_ssize_t size, int flags) goto error; } - if (n != size) { - _PyBytes_Resize(&bytes, n); - } - - return bytes; + return PyBytesWriter_FinishWithSize(writer, n); error: - Py_DECREF(bytes); + PyBytesWriter_Discard(writer); return NULL; } #endif /* HAVE_GETRANDOM_SYSCALL */ diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index dc8e08ac3e522e..f76be53f6de4ef 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -3467,7 +3467,6 @@ sock_getsockopt(PyObject *self, PyObject *args) int level; int optname; int res; - PyObject *buf; socklen_t buflen = 0; int flag = 0; socklen_t flagsize; @@ -3512,17 +3511,17 @@ sock_getsockopt(PyObject *self, PyObject *args) "getsockopt buflen out of range"); return NULL; } - buf = PyBytes_FromStringAndSize((char *)NULL, buflen); - if (buf == NULL) + PyBytesWriter *writer = PyBytesWriter_Create(buflen); + if (writer == NULL) { return NULL; + } res = getsockopt(get_sock_fd(s), level, optname, - (void *)PyBytes_AS_STRING(buf), &buflen); + PyBytesWriter_GetData(writer), &buflen); if (res < 0) { - Py_DECREF(buf); + PyBytesWriter_Discard(writer); return s->errorhandler(); } - _PyBytes_Resize(&buf, buflen); - return buf; + return PyBytesWriter_FinishWithSize(writer, buflen); } PyDoc_STRVAR(getsockopt_doc, diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index 1ee14e31612860..f1312e687da71c 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -833,22 +833,24 @@ save_unconsumed_input(compobject *self, Py_buffer *data, int err) input data in self->unused_data. */ if (self->zst.avail_in > 0) { Py_ssize_t old_size = PyBytes_GET_SIZE(self->unused_data); - Py_ssize_t new_size, left_size; - PyObject *new_data; + Py_ssize_t left_size; left_size = (Byte *)data->buf + data->len - self->zst.next_in; if (left_size > (PY_SSIZE_T_MAX - old_size)) { PyErr_NoMemory(); return -1; } - new_size = old_size + left_size; - new_data = PyBytes_FromStringAndSize(NULL, new_size); - if (new_data == NULL) + PyBytesWriter *writer = PyBytesWriter_Create(old_size + left_size); + if (writer == NULL) { return -1; - memcpy(PyBytes_AS_STRING(new_data), - PyBytes_AS_STRING(self->unused_data), old_size); - memcpy(PyBytes_AS_STRING(new_data) + old_size, - self->zst.next_in, left_size); - Py_SETREF(self->unused_data, new_data); + } + char *new_data = PyBytesWriter_GetData(writer); + memcpy(new_data, PyBytes_AS_STRING(self->unused_data), old_size); + memcpy(new_data + old_size, self->zst.next_in, left_size); + PyObject *new_unused_data = PyBytesWriter_Finish(writer); + if (new_unused_data == NULL) { + return -1; + } + Py_SETREF(self->unused_data, new_unused_data); self->zst.avail_in = 0; } } @@ -1009,7 +1011,7 @@ zlib_Compress_flush_impl(compobject *self, PyTypeObject *cls, int mode) /* Flushing with Z_NO_FLUSH is a no-op, so there's no point in doing any work at all; just return an empty string. */ if (mode == Z_NO_FLUSH) { - return PyBytes_FromStringAndSize(NULL, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); } ENTER_ZLIB(self); @@ -1764,11 +1766,7 @@ zlib__ZlibDecompressor_impl(PyTypeObject *type, int wbits, PyObject *zdict) self->zst.zfree = PyZlib_Free; self->zst.next_in = NULL; self->zst.avail_in = 0; - self->unused_data = PyBytes_FromStringAndSize(NULL, 0); - if (self->unused_data == NULL) { - Py_CLEAR(self); - return NULL; - } + self->unused_data = Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); self->lock = PyThread_allocate_lock(); if (self->lock == NULL) { Py_DECREF(self); diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index fc9e1bef80f037..7a59c76941254c 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -36,6 +36,9 @@ class bytes "PyBytesObject *" "&PyBytes_Type" /* Forward declaration */ Py_LOCAL_INLINE(Py_ssize_t) _PyBytesWriter_GetSize(_PyBytesWriter *writer, char *str); +static void* _PyBytesWriter_ResizeAndUpdatePointer(PyBytesWriter *writer, + Py_ssize_t size, void *data); +static Py_ssize_t _PyBytesWriter_GetAllocated(PyBytesWriter *writer); #define CHARACTERS _Py_SINGLETON(bytes_characters) @@ -1090,22 +1093,16 @@ PyObject *_PyBytes_DecodeEscape2(const char *s, int *first_invalid_escape_char, const char **first_invalid_escape_ptr) { - int c; - char *p; - const char *end; - _PyBytesWriter writer; - - _PyBytesWriter_Init(&writer); - - p = _PyBytesWriter_Alloc(&writer, len); - if (p == NULL) + PyBytesWriter *writer = PyBytesWriter_Create(len); + if (writer == NULL) { return NULL; - writer.overallocate = 1; + } + char *p = PyBytesWriter_GetData(writer); *first_invalid_escape_char = -1; *first_invalid_escape_ptr = NULL; - end = s + len; + const char *end = s + len; while (s < end) { if (*s != '\\') { *p++ = *s++; @@ -1134,7 +1131,8 @@ PyObject *_PyBytes_DecodeEscape2(const char *s, case 'a': *p++ = '\007'; break; /* BEL, not classic C */ case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': - c = s[-1] - '0'; + { + int c = s[-1] - '0'; if (s < end && '0' <= *s && *s <= '7') { c = (c<<3) + *s++ - '0'; if (s < end && '0' <= *s && *s <= '7') @@ -1149,6 +1147,7 @@ PyObject *_PyBytes_DecodeEscape2(const char *s, } *p++ = c; break; + } case 'x': if (s+1 < end) { int digit1, digit2; @@ -1195,10 +1194,10 @@ PyObject *_PyBytes_DecodeEscape2(const char *s, } } - return _PyBytesWriter_Finish(&writer, p); + return PyBytesWriter_FinishWithPointer(writer, p); failed: - _PyBytesWriter_Dealloc(&writer); + PyBytesWriter_Discard(writer); return NULL; } @@ -2535,17 +2534,13 @@ bytes_fromhex_impl(PyTypeObject *type, PyObject *string) PyObject* _PyBytes_FromHex(PyObject *string, int use_bytearray) { - char *buf; Py_ssize_t hexlen, invalid_char; unsigned int top, bot; const Py_UCS1 *str, *start, *end; - _PyBytesWriter writer; + PyBytesWriter *writer = NULL; Py_buffer view; view.obj = NULL; - _PyBytesWriter_Init(&writer); - writer.use_bytearray = use_bytearray; - if (PyUnicode_Check(string)) { hexlen = PyUnicode_GET_LENGTH(string); @@ -2581,10 +2576,16 @@ _PyBytes_FromHex(PyObject *string, int use_bytearray) } /* This overestimates if there are spaces */ - buf = _PyBytesWriter_Alloc(&writer, hexlen / 2); - if (buf == NULL) { + if (use_bytearray) { + writer = _PyBytesWriter_CreateByteArray(hexlen / 2); + } + else { + writer = PyBytesWriter_Create(hexlen / 2); + } + if (writer == NULL) { goto release_buffer; } + char *buf = PyBytesWriter_GetData(writer); start = str; end = str + hexlen; @@ -2623,7 +2624,7 @@ _PyBytes_FromHex(PyObject *string, int use_bytearray) if (view.obj != NULL) { PyBuffer_Release(&view); } - return _PyBytesWriter_Finish(&writer, buf); + return PyBytesWriter_FinishWithPointer(writer, buf); error: if (invalid_char == -1) { @@ -2634,7 +2635,7 @@ _PyBytes_FromHex(PyObject *string, int use_bytearray) "non-hexadecimal number found in " "fromhex() arg at position %zd", invalid_char); } - _PyBytesWriter_Dealloc(&writer); + PyBytesWriter_Discard(writer); release_buffer: if (view.obj != NULL) { @@ -2789,7 +2790,7 @@ bytes_new_impl(PyTypeObject *type, PyObject *x, const char *encoding, "errors without a string argument"); return NULL; } - bytes = PyBytes_FromStringAndSize(NULL, 0); + bytes = Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); } else if (encoding != NULL) { /* Encode via the codec registry */ @@ -2861,23 +2862,25 @@ bytes_new_impl(PyTypeObject *type, PyObject *x, const char *encoding, static PyObject* _PyBytes_FromBuffer(PyObject *x) { - PyObject *new; Py_buffer view; - if (PyObject_GetBuffer(x, &view, PyBUF_FULL_RO) < 0) return NULL; - new = PyBytes_FromStringAndSize(NULL, view.len); - if (!new) + PyBytesWriter *writer = PyBytesWriter_Create(view.len); + if (writer == NULL) { goto fail; - if (PyBuffer_ToContiguous(((PyBytesObject *)new)->ob_sval, - &view, view.len, 'C') < 0) + } + + if (PyBuffer_ToContiguous(PyBytesWriter_GetData(writer), + &view, view.len, 'C') < 0) { goto fail; + } + PyBuffer_Release(&view); - return new; + return PyBytesWriter_Finish(writer); fail: - Py_XDECREF(new); + PyBytesWriter_Discard(writer); PyBuffer_Release(&view); return NULL; } @@ -2885,23 +2888,18 @@ _PyBytes_FromBuffer(PyObject *x) static PyObject* _PyBytes_FromList(PyObject *x) { - Py_ssize_t i, size = PyList_GET_SIZE(x); - Py_ssize_t value; - char *str; - PyObject *item; - _PyBytesWriter writer; - - _PyBytesWriter_Init(&writer); - str = _PyBytesWriter_Alloc(&writer, size); - if (str == NULL) + Py_ssize_t size = PyList_GET_SIZE(x); + PyBytesWriter *writer = PyBytesWriter_Create(size); + if (writer == NULL) { return NULL; - writer.overallocate = 1; - size = writer.allocated; + } + char *str = PyBytesWriter_GetData(writer); + size = _PyBytesWriter_GetAllocated(writer); - for (i = 0; i < PyList_GET_SIZE(x); i++) { - item = PyList_GET_ITEM(x, i); + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(x); i++) { + PyObject *item = PyList_GET_ITEM(x, i); Py_INCREF(item); - value = PyNumber_AsSsize_t(item, NULL); + Py_ssize_t value = PyNumber_AsSsize_t(item, NULL); Py_DECREF(item); if (value == -1 && PyErr_Occurred()) goto error; @@ -2913,33 +2911,33 @@ _PyBytes_FromList(PyObject *x) } if (i >= size) { - str = _PyBytesWriter_Resize(&writer, str, size+1); - if (str == NULL) - return NULL; - size = writer.allocated; + str = _PyBytesWriter_ResizeAndUpdatePointer(writer, size + 1, str); + if (str == NULL) { + goto error; + } + size = _PyBytesWriter_GetAllocated(writer); } *str++ = (char) value; } - return _PyBytesWriter_Finish(&writer, str); + return PyBytesWriter_FinishWithPointer(writer, str); - error: - _PyBytesWriter_Dealloc(&writer); +error: + PyBytesWriter_Discard(writer); return NULL; } static PyObject* _PyBytes_FromTuple(PyObject *x) { - PyObject *bytes; Py_ssize_t i, size = PyTuple_GET_SIZE(x); Py_ssize_t value; - char *str; PyObject *item; - bytes = PyBytes_FromStringAndSize(NULL, size); - if (bytes == NULL) + PyBytesWriter *writer = PyBytesWriter_Create(size); + if (writer == NULL) { return NULL; - str = ((PyBytesObject *)bytes)->ob_sval; + } + char *str = PyBytesWriter_GetData(writer); for (i = 0; i < size; i++) { item = PyTuple_GET_ITEM(x, i); @@ -2954,31 +2952,29 @@ _PyBytes_FromTuple(PyObject *x) } *str++ = (char) value; } - return bytes; + return PyBytesWriter_Finish(writer); error: - Py_DECREF(bytes); + PyBytesWriter_Discard(writer); return NULL; } static PyObject * _PyBytes_FromIterator(PyObject *it, PyObject *x) { - char *str; Py_ssize_t i, size; - _PyBytesWriter writer; /* For iterator version, create a bytes object and resize as needed */ size = PyObject_LengthHint(x, 64); if (size == -1 && PyErr_Occurred()) return NULL; - _PyBytesWriter_Init(&writer); - str = _PyBytesWriter_Alloc(&writer, size); - if (str == NULL) + PyBytesWriter *writer = PyBytesWriter_Create(size); + if (writer == NULL) { return NULL; - writer.overallocate = 1; - size = writer.allocated; + } + char *str = PyBytesWriter_GetData(writer); + size = _PyBytesWriter_GetAllocated(writer); /* Run the iterator to exhaustion */ for (i = 0; ; i++) { @@ -3008,18 +3004,18 @@ _PyBytes_FromIterator(PyObject *it, PyObject *x) /* Append the byte */ if (i >= size) { - str = _PyBytesWriter_Resize(&writer, str, size+1); - if (str == NULL) - return NULL; - size = writer.allocated; + str = _PyBytesWriter_ResizeAndUpdatePointer(writer, size + 1, str); + if (str == NULL) { + goto error; + } + size = _PyBytesWriter_GetAllocated(writer); } *str++ = (char) value; } - - return _PyBytesWriter_Finish(&writer, str); + return PyBytesWriter_FinishWithPointer(writer, str); error: - _PyBytesWriter_Dealloc(&writer); + PyBytesWriter_Discard(writer); return NULL; } @@ -3684,7 +3680,7 @@ _PyBytesWriter_Finish(_PyBytesWriter *writer, void *str) if (size == 0 && !writer->use_bytearray) { Py_CLEAR(writer->buffer); /* Get the empty byte string singleton */ - result = PyBytes_FromStringAndSize(NULL, 0); + result = Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); } else if (writer->use_small_buffer) { if (writer->use_bytearray) { @@ -3987,6 +3983,13 @@ PyBytesWriter_GetSize(PyBytesWriter *writer) } +static Py_ssize_t +_PyBytesWriter_GetAllocated(PyBytesWriter *writer) +{ + return byteswriter_allocated(writer); +} + + int PyBytesWriter_Resize(PyBytesWriter *writer, Py_ssize_t size) { @@ -4002,6 +4005,18 @@ PyBytesWriter_Resize(PyBytesWriter *writer, Py_ssize_t size) } +static void* +_PyBytesWriter_ResizeAndUpdatePointer(PyBytesWriter *writer, Py_ssize_t size, + void *data) +{ + Py_ssize_t pos = (char*)data - byteswriter_data(writer); + if (PyBytesWriter_Resize(writer, size) < 0) { + return NULL; + } + return byteswriter_data(writer) + pos; +} + + int PyBytesWriter_Grow(PyBytesWriter *writer, Py_ssize_t size) { diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index 94ff0fe624e61e..f1232f389210ea 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -2285,7 +2285,6 @@ memoryview_tobytes_impl(PyMemoryViewObject *self, const char *order) { Py_buffer *src = VIEW_ADDR(self); char ord = 'C'; - PyObject *bytes; CHECK_RELEASED(self); @@ -2303,16 +2302,18 @@ memoryview_tobytes_impl(PyMemoryViewObject *self, const char *order) } } - bytes = PyBytes_FromStringAndSize(NULL, src->len); - if (bytes == NULL) + PyBytesWriter *writer = PyBytesWriter_Create(src->len); + if (writer == NULL) { return NULL; + } - if (PyBuffer_ToContiguous(PyBytes_AS_STRING(bytes), src, src->len, ord) < 0) { - Py_DECREF(bytes); + if (PyBuffer_ToContiguous(PyBytesWriter_GetData(writer), + src, src->len, ord) < 0) { + PyBytesWriter_Discard(writer); return NULL; } - return bytes; + return PyBytesWriter_Finish(writer); } /*[clinic input] @@ -2344,8 +2345,6 @@ memoryview_hex_impl(PyMemoryViewObject *self, PyObject *sep, /*[clinic end generated code: output=430ca760f94f3ca7 input=539f6a3a5fb56946]*/ { Py_buffer *src = VIEW_ADDR(self); - PyObject *bytes; - PyObject *ret; CHECK_RELEASED(self); @@ -2353,19 +2352,22 @@ memoryview_hex_impl(PyMemoryViewObject *self, PyObject *sep, return _Py_strhex_with_sep(src->buf, src->len, sep, bytes_per_sep); } - bytes = PyBytes_FromStringAndSize(NULL, src->len); - if (bytes == NULL) + PyBytesWriter *writer = PyBytesWriter_Create(src->len); + if (writer == NULL) { return NULL; + } - if (PyBuffer_ToContiguous(PyBytes_AS_STRING(bytes), src, src->len, 'C') < 0) { - Py_DECREF(bytes); + if (PyBuffer_ToContiguous(PyBytesWriter_GetData(writer), + src, src->len, 'C') < 0) { + PyBytesWriter_Discard(writer); return NULL; } - ret = _Py_strhex_with_sep( - PyBytes_AS_STRING(bytes), PyBytes_GET_SIZE(bytes), - sep, bytes_per_sep); - Py_DECREF(bytes); + PyObject *ret = _Py_strhex_with_sep( + PyBytesWriter_GetData(writer), + PyBytesWriter_GetSize(writer), + sep, bytes_per_sep); + PyBytesWriter_Discard(writer); return ret; } diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 4c88e4c1fdca2e..c8d2c68615e13e 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -4913,7 +4913,7 @@ _PyUnicode_EncodeUTF7(PyObject *str, len = PyUnicode_GET_LENGTH(str); if (len == 0) - return PyBytes_FromStringAndSize(NULL, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); /* It might be possible to tighten this worst case */ if (len > PY_SSIZE_T_MAX / 8) @@ -6914,7 +6914,7 @@ PyUnicode_AsUnicodeEscapeString(PyObject *unicode) len = PyUnicode_GET_LENGTH(unicode); if (len == 0) { - return PyBytes_FromStringAndSize(NULL, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); } kind = PyUnicode_KIND(unicode); @@ -7364,7 +7364,7 @@ unicode_encode_ucs1(PyObject *unicode, /* allocate enough for a simple encoding without replacements, if we need more, we'll resize */ if (size == 0) - return PyBytes_FromStringAndSize(NULL, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); _PyBytesWriter_Init(&writer); str = _PyBytesWriter_Alloc(&writer, size); @@ -8305,7 +8305,7 @@ encode_code_page(int code_page, } if (len == 0) - return PyBytes_FromStringAndSize(NULL, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); offset = 0; do