Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions Include/cpython/pystate.h
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,11 @@ struct _ts {
*/
unsigned long native_thread_id;

/* List of objects that still need to be cleaned up, singly linked
* via their gc headers' gc_next pointers. The list is populated by
* _PyTrash_thread_deposit_object and cleaned up by
* _PyTrash_thread_destroy_chain.
*/
PyObject *delete_later;

/* Tagged pointer to top-most critical section, or zero if there is no
Expand Down
9 changes: 3 additions & 6 deletions Include/internal/pycore_interp_structs.h
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ extern "C" {
#include "pycore_structs.h" // PyHamtObject
#include "pycore_tstate.h" // _PyThreadStateImpl
#include "pycore_typedefs.h" // _PyRuntimeState
#include "pycore_uop.h" // struct _PyUOpInstruction


#define CODE_MAX_WATCHERS 8
Expand Down Expand Up @@ -202,12 +203,6 @@ enum _GCPhase {
#define NUM_GENERATIONS 3

struct _gc_runtime_state {
/* List of objects that still need to be cleaned up, singly linked
* via their gc headers' gc_prev pointers. */
PyObject *trash_delete_later;
/* Current call-stack depth of tp_dealloc calls. */
int trash_delete_nesting;

/* Is automatic collection enabled? */
int enabled;
int debug;
Expand Down Expand Up @@ -949,6 +944,8 @@ struct _is {
struct callable_cache callable_cache;
PyObject *common_consts[NUM_COMMON_CONSTANTS];
bool jit;
bool compiling;
struct _PyUOpInstruction *jit_uop_buffer;
struct _PyExecutorObject *executor_list_head;
struct _PyExecutorObject *executor_deletion_list_head;
struct _PyExecutorObject *cold_executor;
Expand Down
30 changes: 1 addition & 29 deletions Include/internal/pycore_optimizer.h
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ extern "C" {
#endif

#include "pycore_typedefs.h" // _PyInterpreterFrame
#include "pycore_uop.h" // _PyUOpInstruction
#include "pycore_uop_ids.h"
#include "pycore_stackref.h" // _PyStackRef
#include <stdbool.h>
Expand Down Expand Up @@ -41,32 +42,6 @@ typedef struct {
PyCodeObject *code; // Weak (NULL if no corresponding ENTER_EXECUTOR).
} _PyVMData;

/* Depending on the format,
* the 32 bits between the oparg and operand are:
* UOP_FORMAT_TARGET:
* uint32_t target;
* UOP_FORMAT_JUMP
* uint16_t jump_target;
* uint16_t error_target;
*/
typedef struct {
uint16_t opcode:15;
uint16_t format:1;
uint16_t oparg;
union {
uint32_t target;
struct {
uint16_t jump_target;
uint16_t error_target;
};
};
uint64_t operand0; // A cache entry
uint64_t operand1;
#ifdef Py_STATS
uint64_t execution_count;
#endif
} _PyUOpInstruction;

typedef struct _PyExitData {
uint32_t target;
uint16_t index;
Expand Down Expand Up @@ -118,9 +93,6 @@ PyAPI_FUNC(void) _Py_Executors_InvalidateCold(PyInterpreterState *interp);
// trace_run_counter is greater than this value.
#define JIT_CLEANUP_THRESHOLD 100000

// This is the length of the trace we project initially.
#define UOP_MAX_TRACE_LENGTH 1200

#define TRACE_STACK_SIZE 5

int _Py_uop_analyze_and_optimize(_PyInterpreterFrame *frame,
Expand Down
45 changes: 45 additions & 0 deletions Include/internal/pycore_uop.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
#ifndef Py_CORE_UOP_H
#define Py_CORE_UOP_H
#ifdef __cplusplus
extern "C" {
#endif

#ifndef Py_BUILD_CORE
# error "this header requires Py_BUILD_CORE define"
#endif

#include <stdint.h>
/* Depending on the format,
* the 32 bits between the oparg and operand are:
* UOP_FORMAT_TARGET:
* uint32_t target;
* UOP_FORMAT_JUMP
* uint16_t jump_target;
* uint16_t error_target;
*/
typedef struct _PyUOpInstruction{
uint16_t opcode:15;
uint16_t format:1;
uint16_t oparg;
union {
uint32_t target;
struct {
uint16_t jump_target;
uint16_t error_target;
};
};
uint64_t operand0; // A cache entry
uint64_t operand1;
#ifdef Py_STATS
uint64_t execution_count;
#endif
} _PyUOpInstruction;

// This is the length of the trace we project initially.
#define UOP_MAX_TRACE_LENGTH 1200
#define UOP_BUFFER_SIZE (UOP_MAX_TRACE_LENGTH * sizeof(_PyUOpInstruction))

#ifdef __cplusplus
}
#endif
#endif /* !Py_INTERNAL_UOP_H */
2 changes: 1 addition & 1 deletion Lib/multiprocessing/dummy/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@

class DummyProcess(threading.Thread):

def __init__(self, group=None, target=None, name=None, args=(), kwargs={}):
def __init__(self, group=None, target=None, name=None, args=(), kwargs=None):
threading.Thread.__init__(self, group, target, name, args, kwargs)
self._pid = None
self._children = weakref.WeakKeyDictionary()
Expand Down
4 changes: 2 additions & 2 deletions Lib/multiprocessing/process.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ class BaseProcess(object):
def _Popen(self):
raise NotImplementedError

def __init__(self, group=None, target=None, name=None, args=(), kwargs={},
def __init__(self, group=None, target=None, name=None, args=(), kwargs=None,
*, daemon=None):
assert group is None, 'group argument must be None for now'
count = next(_process_counter)
Expand All @@ -89,7 +89,7 @@ def __init__(self, group=None, target=None, name=None, args=(), kwargs={},
self._closed = False
self._target = target
self._args = tuple(args)
self._kwargs = dict(kwargs)
self._kwargs = dict(kwargs) if kwargs else {}
self._name = name or type(self).__name__ + '-' + \
':'.join(str(i) for i in self._identity)
if daemon is not None:
Expand Down
17 changes: 17 additions & 0 deletions Lib/test/_test_multiprocessing.py
Original file line number Diff line number Diff line change
Expand Up @@ -5278,6 +5278,23 @@ def test_invalid_handles(self):
multiprocessing.connection.Connection, -1)


#
# Regression tests for BaseProcess kwargs handling
#

class TestBaseProcessKwargs(unittest.TestCase):
def test_default_kwargs_not_shared_between_instances(self):
# Creating multiple Process instances without passing kwargs
# must create independent empty dicts (no shared state).
p1 = multiprocessing.Process(target=lambda: None)
p2 = multiprocessing.Process(target=lambda: None)
self.assertIsInstance(p1._kwargs, dict)
self.assertIsInstance(p2._kwargs, dict)
self.assertIsNot(p1._kwargs, p2._kwargs)
# Mutating one should not affect the other
p1._kwargs['x'] = 1
self.assertNotIn('x', p2._kwargs)


@hashlib_helper.requires_hashdigest('sha256')
class OtherTest(unittest.TestCase):
Expand Down
4 changes: 3 additions & 1 deletion Lib/test/test_math.py
Original file line number Diff line number Diff line change
Expand Up @@ -1485,7 +1485,9 @@ def __rmul__(self, other):

# Error cases that arose during development
args = ((-5, -5, 10), (1.5, 4611686018427387904, 2305843009213693952))
self.assertEqual(sumprod(*args), 0.0)
self.assertEqual(sumprod(*args), -7.5)
self.assertEqual(sumprod([-0.01, 1, -1, 0.01], [1, 1, 1, 1]), 0.0)
self.assertEqual(sumprod([1, 1, 1, 1], [-0.01, 1, -1, 0.01], ), 0.0)

@requires_IEEE_754
@unittest.skipIf(HAVE_DOUBLE_ROUNDING,
Expand Down
1 change: 1 addition & 0 deletions Makefile.pre.in
Original file line number Diff line number Diff line change
Expand Up @@ -1435,6 +1435,7 @@ PYTHON_HEADERS= \
$(srcdir)/Include/internal/pycore_unicodeobject_generated.h \
$(srcdir)/Include/internal/pycore_unionobject.h \
$(srcdir)/Include/internal/pycore_uniqueid.h \
$(srcdir)/Include/internal/pycore_uop.h \
$(srcdir)/Include/internal/pycore_uop_ids.h \
$(srcdir)/Include/internal/pycore_uop_metadata.h \
$(srcdir)/Include/internal/pycore_warnings.h \
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Fixed a missing case in :func:`math.sumprod` where a low precision path was
taken when an int/int input pair followed a float input.
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
:class:`!multiprocessing.BaseProcess` defaults ``kwargs`` to ``None`` instead of a shared dictionary.
29 changes: 14 additions & 15 deletions Modules/mathmodule.c
Original file line number Diff line number Diff line change
Expand Up @@ -2937,32 +2937,31 @@ math_sumprod_impl(PyObject *module, PyObject *p, PyObject *q)

if (!finished) {
double flt_p, flt_q;
bool p_type_float = PyFloat_CheckExact(p_i);
bool q_type_float = PyFloat_CheckExact(q_i);
if (p_type_float && q_type_float) {
flt_p = PyFloat_AS_DOUBLE(p_i);
flt_q = PyFloat_AS_DOUBLE(q_i);
} else if (p_type_float && (PyLong_CheckExact(q_i) || PyBool_Check(q_i))) {
/* We care about float/int pairs and int/float pairs because
they arise naturally in several use cases such as price
times quantity, measurements with integer weights, or
data selected by a vector of bools. */

if (PyFloat_CheckExact(p_i)) {
flt_p = PyFloat_AS_DOUBLE(p_i);
flt_q = PyLong_AsDouble(q_i);
if (flt_q == -1.0 && PyErr_Occurred()) {
} else if (PyLong_CheckExact(p_i) || PyBool_Check(p_i)) {
flt_p = PyLong_AsDouble(p_i);
if (flt_p == -1.0 && PyErr_Occurred()) {
PyErr_Clear();
goto finalize_flt_path;
}
} else if (q_type_float && (PyLong_CheckExact(p_i) || PyBool_Check(p_i))) {
} else {
goto finalize_flt_path;
}

if (PyFloat_CheckExact(q_i)) {
flt_q = PyFloat_AS_DOUBLE(q_i);
flt_p = PyLong_AsDouble(p_i);
if (flt_p == -1.0 && PyErr_Occurred()) {
} else if (PyLong_CheckExact(q_i) || PyBool_Check(q_i)) {
flt_q = PyLong_AsDouble(q_i);
if (flt_q == -1.0 && PyErr_Occurred()) {
PyErr_Clear();
goto finalize_flt_path;
}
} else {
goto finalize_flt_path;
}

TripleLength new_flt_total = tl_fma(flt_p, flt_q, flt_total);
if (isfinite(new_flt_total.hi)) {
flt_total = new_flt_total;
Expand Down
4 changes: 2 additions & 2 deletions Objects/object.c
Original file line number Diff line number Diff line change
Expand Up @@ -3051,7 +3051,7 @@ Py_ReprLeave(PyObject *obj)

/* Trashcan support. */

/* Add op to the gcstate->trash_delete_later list. Called when the current
/* Add op to the tstate->delete_later list. Called when the current
* call-stack depth gets large. op must be a gc'ed object, with refcount 0.
* Py_DECREF must already have been called on it.
*/
Expand All @@ -3077,7 +3077,7 @@ _PyTrash_thread_deposit_object(PyThreadState *tstate, PyObject *op)
tstate->delete_later = op;
}

/* Deallocate all the objects in the gcstate->trash_delete_later list.
/* Deallocate all the objects in the tstate->delete_later list.
* Called when the call-stack unwinds again. */
void
_PyTrash_thread_destroy_chain(PyThreadState *tstate)
Expand Down
9 changes: 9 additions & 0 deletions PCbuild/pythoncore.vcxproj.filters
Original file line number Diff line number Diff line change
Expand Up @@ -882,6 +882,15 @@
<ClInclude Include="..\Include\internal\pycore_uniqueid.h">
<Filter>Include\internal</Filter>
</ClInclude>
<ClInclude Include="..\Include\internal\pycore_uop.h">
<Filter>Include\internal</Filter>
</ClInclude>
<ClInclude Include="..\Include\internal\pycore_uop_ids.h">
<Filter>Include\internal</Filter>
</ClInclude>
<ClInclude Include="..\Include\internal\pycore_uop_metadata.h">
<Filter>Include\internal</Filter>
</ClInclude>
<ClInclude Include="..\Include\internal\mimalloc\mimalloc.h">
<Filter>Include\internal\mimalloc</Filter>
</ClInclude>
Expand Down
18 changes: 16 additions & 2 deletions Python/optimizer.c
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,10 @@ _PyOptimizer_Optimize(
_PyExecutorObject **executor_ptr, int chain_depth)
{
_PyStackRef *stack_pointer = frame->stackpointer;
assert(_PyInterpreterState_GET()->jit);
PyInterpreterState *interp = _PyInterpreterState_GET();
assert(interp->jit);
assert(!interp->compiling);
interp->compiling = true;
// The first executor in a chain and the MAX_CHAIN_DEPTH'th executor *must*
// make progress in order to avoid infinite loops or excessively-long
// side-exit chains. We can only insert the executor into the bytecode if
Expand All @@ -126,10 +129,12 @@ _PyOptimizer_Optimize(
PyCodeObject *code = _PyFrame_GetCode(frame);
assert(PyCode_Check(code));
if (progress_needed && !has_space_for_executor(code, start)) {
interp->compiling = false;
return 0;
}
int err = uop_optimize(frame, start, executor_ptr, (int)(stack_pointer - _PyFrame_Stackbase(frame)), progress_needed);
if (err <= 0) {
interp->compiling = false;
return err;
}
assert(*executor_ptr != NULL);
Expand All @@ -143,6 +148,7 @@ _PyOptimizer_Optimize(
* it might get confused by the executor disappearing,
* but there is not much we can do about that here. */
Py_DECREF(*executor_ptr);
interp->compiling = false;
return 0;
}
insert_executor(code, start, index, *executor_ptr);
Expand All @@ -152,6 +158,7 @@ _PyOptimizer_Optimize(
}
(*executor_ptr)->vm_data.chain_depth = chain_depth;
assert((*executor_ptr)->vm_data.valid);
interp->compiling = false;
return 1;
}

Expand Down Expand Up @@ -1280,7 +1287,14 @@ uop_optimize(
{
_PyBloomFilter dependencies;
_Py_BloomFilter_Init(&dependencies);
_PyUOpInstruction buffer[UOP_MAX_TRACE_LENGTH];
PyInterpreterState *interp = _PyInterpreterState_GET();
if (interp->jit_uop_buffer == NULL) {
interp->jit_uop_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE);
if (interp->jit_uop_buffer == NULL) {
return 0;
}
}
_PyUOpInstruction *buffer = interp->jit_uop_buffer;
OPT_STAT_INC(attempts);
char *env_var = Py_GETENV("PYTHON_UOPS_OPTIMIZE");
bool is_noopt = true;
Expand Down
1 change: 1 addition & 0 deletions Python/pylifecycle.c
Original file line number Diff line number Diff line change
Expand Up @@ -1702,6 +1702,7 @@ finalize_modules(PyThreadState *tstate)

// Invalidate all executors and turn off JIT:
interp->jit = false;
interp->compiling = false;
#ifdef _Py_TIER2
_Py_Executors_InvalidateAll(interp, 0);
#endif
Expand Down
11 changes: 11 additions & 0 deletions Python/pystate.c
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
#include "pycore_runtime_init.h" // _PyRuntimeState_INIT
#include "pycore_stackref.h" // Py_STACKREF_DEBUG
#include "pycore_time.h" // _PyTime_Init()
#include "pycore_uop.h" // UOP_BUFFER_SIZE
#include "pycore_uniqueid.h" // _PyObject_FinalizePerThreadRefcounts()


Expand Down Expand Up @@ -550,6 +551,11 @@ init_interpreter(PyInterpreterState *interp,
#ifdef Py_GIL_DISABLED
_Py_brc_init_state(interp);
#endif

#ifdef _Py_TIER2
// Ensure the buffer is to be set as NULL.
interp->jit_uop_buffer = NULL;
#endif
llist_init(&interp->mem_free_queue.head);
llist_init(&interp->asyncio_tasks_head);
interp->asyncio_tasks_lock = (PyMutex){0};
Expand All @@ -565,6 +571,7 @@ init_interpreter(PyInterpreterState *interp,
}
interp->_code_object_generation = 0;
interp->jit = false;
interp->compiling = false;
interp->executor_list_head = NULL;
interp->executor_deletion_list_head = NULL;
interp->executor_deletion_list_remaining_capacity = 0;
Expand Down Expand Up @@ -797,6 +804,10 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate)

#ifdef _Py_TIER2
_Py_ClearExecutorDeletionList(interp);
if (interp->jit_uop_buffer != NULL) {
_PyObject_VirtualFree(interp->jit_uop_buffer, UOP_BUFFER_SIZE);
interp->jit_uop_buffer = NULL;
}
#endif
_PyAST_Fini(interp);
_PyAtExit_Fini(interp);
Expand Down
Loading