From 1c0c832b20015e0c17043d53ff6d29bb8ee20d52 Mon Sep 17 00:00:00 2001 From: Mohamed Koubaa Date: Tue, 7 Jul 2020 16:11:01 -0500 Subject: [PATCH 1/6] port multiprocessing to multiphase init --- ...2020-07-07-16-10-52.bpo-1635741.zU-H_n.rst | 1 + Modules/_multiprocessing/multiprocessing.c | 53 +++++++++---------- 2 files changed, 27 insertions(+), 27 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-07-07-16-10-52.bpo-1635741.zU-H_n.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-07-07-16-10-52.bpo-1635741.zU-H_n.rst b/Misc/NEWS.d/next/Core and Builtins/2020-07-07-16-10-52.bpo-1635741.zU-H_n.rst new file mode 100644 index 00000000000000..52e184dc317074 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-07-07-16-10-52.bpo-1635741.zU-H_n.rst @@ -0,0 +1 @@ +Port :mod:`multiprocessing` to multi-phase initialization diff --git a/Modules/_multiprocessing/multiprocessing.c b/Modules/_multiprocessing/multiprocessing.c index 806e638340f7af..58ed8575f8f769 100644 --- a/Modules/_multiprocessing/multiprocessing.c +++ b/Modules/_multiprocessing/multiprocessing.c @@ -139,34 +139,15 @@ static PyMethodDef module_methods[] = { * Initialize */ -static struct PyModuleDef multiprocessing_module = { - PyModuleDef_HEAD_INIT, - "_multiprocessing", - NULL, - -1, - module_methods, - NULL, - NULL, - NULL, - NULL -}; - - -PyMODINIT_FUNC -PyInit__multiprocessing(void) +int multiprocessing_exec(PyObject *module) { - PyObject *module, *temp, *value = NULL; - - /* Initialize module */ - module = PyModule_Create(&multiprocessing_module); - if (!module) - return NULL; + PyObject *value = NULL; #if defined(MS_WINDOWS) || \ (defined(HAVE_SEM_OPEN) && !defined(POSIX_SEMAPHORES_NOT_ENABLED)) /* Add _PyMp_SemLock type to module */ if (PyType_Ready(&_PyMp_SemLockType) < 0) - return NULL; + return -1; Py_INCREF(&_PyMp_SemLockType); { PyObject *py_sem_value_max; @@ -180,7 +161,7 @@ PyInit__multiprocessing(void) else py_sem_value_max = PyLong_FromLong(SEM_VALUE_MAX); if (py_sem_value_max == NULL) - return NULL; + return -1; PyDict_SetItemString(_PyMp_SemLockType.tp_dict, "SEM_VALUE_MAX", py_sem_value_max); } @@ -188,9 +169,9 @@ PyInit__multiprocessing(void) #endif /* Add configuration macros */ - temp = PyDict_New(); + PyObject *temp = PyDict_New(); if (!temp) - return NULL; + return -1; #define ADD_FLAG(name) \ value = Py_BuildValue("i", name); \ @@ -213,7 +194,25 @@ PyInit__multiprocessing(void) #endif if (PyModule_AddObject(module, "flags", temp) < 0) - return NULL; + return -1; - return module; + return 0; +} + +static PyModuleDef_Slot multiprocessing_slots[] = { + {Py_mod_exec, multiprocessing_exec}, + {0, NULL} +}; + +static struct PyModuleDef multiprocessing_module = { + PyModuleDef_HEAD_INIT, + .m_name = "_multiprocessing", + .m_methods = module_methods, + .m_slots = multiprocessing_slots, +}; + +PyMODINIT_FUNC +PyInit__multiprocessing(void) +{ + return PyModuleDef_Init(&multiprocessing_module); } From b31425dcd4f90bb0eb20266a992fc03055d9de25 Mon Sep 17 00:00:00 2001 From: = <=> Date: Tue, 7 Jul 2020 21:14:03 -0500 Subject: [PATCH 2/6] fix issues --- Modules/_multiprocessing/multiprocessing.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/_multiprocessing/multiprocessing.c b/Modules/_multiprocessing/multiprocessing.c index 58ed8575f8f769..dccfffbb2fae8b 100644 --- a/Modules/_multiprocessing/multiprocessing.c +++ b/Modules/_multiprocessing/multiprocessing.c @@ -139,7 +139,7 @@ static PyMethodDef module_methods[] = { * Initialize */ -int multiprocessing_exec(PyObject *module) +static int multiprocessing_exec(PyObject *module) { PyObject *value = NULL; @@ -175,9 +175,9 @@ int multiprocessing_exec(PyObject *module) #define ADD_FLAG(name) \ value = Py_BuildValue("i", name); \ - if (value == NULL) { Py_DECREF(temp); return NULL; } \ + if (value == NULL) { Py_DECREF(temp); return -1; } \ if (PyDict_SetItemString(temp, #name, value) < 0) { \ - Py_DECREF(temp); Py_DECREF(value); return NULL; } \ + Py_DECREF(temp); Py_DECREF(value); return -1; } \ Py_DECREF(value) #if defined(HAVE_SEM_OPEN) && !defined(POSIX_SEMAPHORES_NOT_ENABLED) From d196df9941886e611b49a8e8643859ec057e8607 Mon Sep 17 00:00:00 2001 From: Mohamed Koubaa Date: Thu, 9 Jul 2020 01:49:04 -0500 Subject: [PATCH 3/6] update based on review --- Modules/_multiprocessing/multiprocessing.c | 50 ++++++++++++++-------- 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/Modules/_multiprocessing/multiprocessing.c b/Modules/_multiprocessing/multiprocessing.c index dccfffbb2fae8b..9906be94628f37 100644 --- a/Modules/_multiprocessing/multiprocessing.c +++ b/Modules/_multiprocessing/multiprocessing.c @@ -139,16 +139,12 @@ static PyMethodDef module_methods[] = { * Initialize */ -static int multiprocessing_exec(PyObject *module) +static int +multiprocessing_exec(PyObject *module) { - PyObject *value = NULL; - #if defined(MS_WINDOWS) || \ (defined(HAVE_SEM_OPEN) && !defined(POSIX_SEMAPHORES_NOT_ENABLED)) - /* Add _PyMp_SemLock type to module */ - if (PyType_Ready(&_PyMp_SemLockType) < 0) - return -1; - Py_INCREF(&_PyMp_SemLockType); + { PyObject *py_sem_value_max; /* Some systems define SEM_VALUE_MAX as an unsigned value that @@ -160,25 +156,42 @@ static int multiprocessing_exec(PyObject *module) py_sem_value_max = PyLong_FromLong(INT_MAX); else py_sem_value_max = PyLong_FromLong(SEM_VALUE_MAX); - if (py_sem_value_max == NULL) + + if (py_sem_value_max == NULL) { + Py_DECREF(py_sem_value_max); return -1; + } PyDict_SetItemString(_PyMp_SemLockType.tp_dict, "SEM_VALUE_MAX", py_sem_value_max); } - PyModule_AddObject(module, "SemLock", (PyObject*)&_PyMp_SemLockType); + + /* Add _PyMp_SemLock type to module */ + Py_INCREF(&_PyMp_SemLockType); + if (PyModule_AddType(module, &_PyMp_SemLockType) < 0) { + Py_DECREF(&_PyMp_SemLockType); + return -1; + } + #endif /* Add configuration macros */ - PyObject *temp = PyDict_New(); - if (!temp) + PyObject *flags = PyDict_New(); + if (!flags) { return -1; + } -#define ADD_FLAG(name) \ - value = Py_BuildValue("i", name); \ - if (value == NULL) { Py_DECREF(temp); return -1; } \ - if (PyDict_SetItemString(temp, #name, value) < 0) { \ - Py_DECREF(temp); Py_DECREF(value); return -1; } \ - Py_DECREF(value) +#define ADD_FLAG(name) { \ + PyObject *value = PyLong_FromLong(name); \ + if (value == NULL) { \ + Py_DECREF(flags); \ + return -1; } \ + } \ + if (PyDict_SetItemString(flags, #name, value) < 0) { \ + Py_DECREF(flags); \ + Py_DECREF(value); \ + return -1; } \ + Py_DECREF(value) \ + } #if defined(HAVE_SEM_OPEN) && !defined(POSIX_SEMAPHORES_NOT_ENABLED) ADD_FLAG(HAVE_SEM_OPEN); @@ -193,8 +206,9 @@ static int multiprocessing_exec(PyObject *module) ADD_FLAG(HAVE_BROKEN_SEM_UNLINK); #endif - if (PyModule_AddObject(module, "flags", temp) < 0) + if (PyModule_AddObject(module, "flags", flags) < 0) { return -1; + } return 0; } From e2a12af665caaf90aeb070e75290705ebfa0e9e0 Mon Sep 17 00:00:00 2001 From: Mohamed Koubaa Date: Thu, 9 Jul 2020 09:38:31 -0500 Subject: [PATCH 4/6] update based on review --- Modules/_multiprocessing/multiprocessing.c | 26 +++++++++++++--------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/Modules/_multiprocessing/multiprocessing.c b/Modules/_multiprocessing/multiprocessing.c index 9906be94628f37..5fd6c4543896ce 100644 --- a/Modules/_multiprocessing/multiprocessing.c +++ b/Modules/_multiprocessing/multiprocessing.c @@ -145,6 +145,11 @@ multiprocessing_exec(PyObject *module) #if defined(MS_WINDOWS) || \ (defined(HAVE_SEM_OPEN) && !defined(POSIX_SEMAPHORES_NOT_ENABLED)) + /* Add _PyMp_SemLock type to module */ + if (PyModule_AddType(module, &_PyMp_SemLockType) < 0) { + return -1; + } + { PyObject *py_sem_value_max; /* Some systems define SEM_VALUE_MAX as an unsigned value that @@ -161,15 +166,12 @@ multiprocessing_exec(PyObject *module) Py_DECREF(py_sem_value_max); return -1; } - PyDict_SetItemString(_PyMp_SemLockType.tp_dict, "SEM_VALUE_MAX", - py_sem_value_max); - } - - /* Add _PyMp_SemLock type to module */ - Py_INCREF(&_PyMp_SemLockType); - if (PyModule_AddType(module, &_PyMp_SemLockType) < 0) { - Py_DECREF(&_PyMp_SemLockType); - return -1; + if (PyDict_SetItemString(_PyMp_SemLockType.tp_dict, "SEM_VALUE_MAX", + py_sem_value_max) < 0) { + Py_DECREF(_PyMp_SemLockType.tp_dict); + Py_DECREF(py_sem_value_max); + return -1; + } } #endif @@ -180,7 +182,8 @@ multiprocessing_exec(PyObject *module) return -1; } -#define ADD_FLAG(name) { \ +#define ADD_FLAG(name) \ + do { \ PyObject *value = PyLong_FromLong(name); \ if (value == NULL) { \ Py_DECREF(flags); \ @@ -191,7 +194,7 @@ multiprocessing_exec(PyObject *module) Py_DECREF(value); \ return -1; } \ Py_DECREF(value) \ - } + } while (0) #if defined(HAVE_SEM_OPEN) && !defined(POSIX_SEMAPHORES_NOT_ENABLED) ADD_FLAG(HAVE_SEM_OPEN); @@ -207,6 +210,7 @@ multiprocessing_exec(PyObject *module) #endif if (PyModule_AddObject(module, "flags", flags) < 0) { + Py_DECREF(flags); return -1; } From 08aa12b5a8be4f0f379855f93ff48fb63b9b9b78 Mon Sep 17 00:00:00 2001 From: Mohamed Koubaa Date: Thu, 9 Jul 2020 10:13:57 -0500 Subject: [PATCH 5/6] fix macro --- Modules/_multiprocessing/multiprocessing.c | 25 +++++++++++----------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/Modules/_multiprocessing/multiprocessing.c b/Modules/_multiprocessing/multiprocessing.c index 5fd6c4543896ce..8f0562e50e871d 100644 --- a/Modules/_multiprocessing/multiprocessing.c +++ b/Modules/_multiprocessing/multiprocessing.c @@ -182,18 +182,19 @@ multiprocessing_exec(PyObject *module) return -1; } -#define ADD_FLAG(name) \ - do { \ - PyObject *value = PyLong_FromLong(name); \ - if (value == NULL) { \ - Py_DECREF(flags); \ - return -1; } \ - } \ - if (PyDict_SetItemString(flags, #name, value) < 0) { \ - Py_DECREF(flags); \ - Py_DECREF(value); \ - return -1; } \ - Py_DECREF(value) \ +#define ADD_FLAG(name) \ + do { \ + PyObject *value = PyLong_FromLong(name); \ + if (value == NULL) { \ + Py_DECREF(flags); \ + return -1; \ + } \ + if (PyDict_SetItemString(flags, #name, value) < 0) { \ + Py_DECREF(flags); \ + Py_DECREF(value); \ + return -1; \ + } \ + Py_DECREF(value); \ } while (0) #if defined(HAVE_SEM_OPEN) && !defined(POSIX_SEMAPHORES_NOT_ENABLED) From 064d8e0ef1dd57577286a5e0530570b91bf54e52 Mon Sep 17 00:00:00 2001 From: Mohamed Koubaa Date: Mon, 10 Aug 2020 08:47:40 -0500 Subject: [PATCH 6/6] fix refcount issue --- Modules/_multiprocessing/multiprocessing.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_multiprocessing/multiprocessing.c b/Modules/_multiprocessing/multiprocessing.c index 8f0562e50e871d..d77d4d374d6ebb 100644 --- a/Modules/_multiprocessing/multiprocessing.c +++ b/Modules/_multiprocessing/multiprocessing.c @@ -168,10 +168,10 @@ multiprocessing_exec(PyObject *module) } if (PyDict_SetItemString(_PyMp_SemLockType.tp_dict, "SEM_VALUE_MAX", py_sem_value_max) < 0) { - Py_DECREF(_PyMp_SemLockType.tp_dict); Py_DECREF(py_sem_value_max); return -1; } + Py_DECREF(py_sem_value_max); } #endif