From 4e839f47c1f79c2302f219b948a3e81d340eb9c2 Mon Sep 17 00:00:00 2001 From: LiYuRio Date: Wed, 3 Jan 2024 20:16:34 +0800 Subject: [PATCH] fix tensor math method inplace converter --- paddle/fluid/pybind/eager_math_op_patch.cc | 803 +++++++++++++-------- python/paddle/optimizer/optimizer.py | 4 +- 2 files changed, 486 insertions(+), 321 deletions(-) diff --git a/paddle/fluid/pybind/eager_math_op_patch.cc b/paddle/fluid/pybind/eager_math_op_patch.cc index 2c01e122914aa4..cdfc882118401b 100644 --- a/paddle/fluid/pybind/eager_math_op_patch.cc +++ b/paddle/fluid/pybind/eager_math_op_patch.cc @@ -233,24 +233,34 @@ static PyObject* tensor__add__method(TensorObject* self, // 2. create or get tensor for other_obj paddle::Tensor other_tensor; if (PyCheckTensor(other_obj)) { - other_tensor = CastPyArg2Tensor(other_obj, 0); - } else if (IsNumpyArray(other_obj)) { - py::object numpy_value = py::object(py::handle(other_obj), true); - other_tensor = paddle::Tensor(place); - InitTensorWithNumpyValue(numpy_value, place, &other_tensor); + auto& self_tensor_ref = self->tensor; + auto& other_tensor_ref = CastPyArg2Tensor(other_obj, 0); + const phi::distributed::ProcessMesh* mesh = nullptr; + if (InputsContainDistTensor(&mesh, self_tensor_ref, other_tensor_ref)) { + ConvertAllInputsToDistTensor(mesh, self_tensor_ref, other_tensor_ref); + } + self_tensor = self_tensor_ref; + other_tensor = other_tensor_ref; } else { - paddle::experimental::Scalar value = - CastPyArg2Scalar(other_obj, "__add__", 0); - { - eager_gil_scoped_release guard; - other_tensor = full_ad_func( - self_tensor.shape(), value, self_tensor.dtype(), self_tensor.place()); + if (IsNumpyArray(other_obj)) { + py::object numpy_value = py::object(py::handle(other_obj), true); + other_tensor = paddle::Tensor(place); + InitTensorWithNumpyValue(numpy_value, place, &other_tensor); + } else { + paddle::experimental::Scalar value = + CastPyArg2Scalar(other_obj, "__add__", 0); + { + eager_gil_scoped_release guard; + other_tensor = full_ad_func(self_tensor.shape(), + value, + self_tensor.dtype(), + self_tensor.place()); + } + } + const phi::distributed::ProcessMesh* mesh = nullptr; + if (InputsContainDistTensor(&mesh, self_tensor, other_tensor)) { + ConvertAllInputsToDistTensor(mesh, self_tensor, other_tensor); } - } - - const phi::distributed::ProcessMesh* mesh = nullptr; - if (InputsContainDistTensor(&mesh, self_tensor, other_tensor)) { - ConvertAllInputsToDistTensor(mesh, self_tensor, other_tensor); } // 3. promote types or unify right var type to left var, float type promotion @@ -313,8 +323,8 @@ static PyObject* tensor__sub__method(TensorObject* self, SetDevice(place); paddle::Tensor ret; - paddle::Tensor self_tensor = self->tensor; + paddle::Tensor self_tensor = self->tensor; PyObject* other_obj = PyTuple_GET_ITEM(args, 0); // 1. scalar exists cases if (PyFloat_Check(other_obj) || PyCheckInteger(other_obj) || @@ -337,27 +347,38 @@ static PyObject* tensor__sub__method(TensorObject* self, return ToPyObject(ret); } + // 2. create or get tensor for other_obj paddle::Tensor other_tensor; if (PyCheckTensor(other_obj)) { - other_tensor = CastPyArg2Tensor(other_obj, 0); - } else if (IsNumpyArray(other_obj)) { - py::object numpy_value = py::object(py::handle(other_obj), true); - other_tensor = paddle::Tensor(place); - InitTensorWithNumpyValue(numpy_value, place, &other_tensor); + auto& self_tensor_ref = self->tensor; + auto& other_tensor_ref = CastPyArg2Tensor(other_obj, 0); + const phi::distributed::ProcessMesh* mesh = nullptr; + if (InputsContainDistTensor(&mesh, self_tensor, other_tensor_ref)) { + ConvertAllInputsToDistTensor(mesh, self_tensor, other_tensor_ref); + } + self_tensor = self_tensor_ref; + other_tensor = other_tensor_ref; } else { - paddle::experimental::Scalar value = - CastPyArg2Scalar(other_obj, "__sub__", 0); - { - eager_gil_scoped_release guard; - other_tensor = full_ad_func( - self_tensor.shape(), value, self_tensor.dtype(), self_tensor.place()); + if (IsNumpyArray(other_obj)) { + py::object numpy_value = py::object(py::handle(other_obj), true); + other_tensor = paddle::Tensor(place); + InitTensorWithNumpyValue(numpy_value, place, &other_tensor); + } else { + paddle::experimental::Scalar value = + CastPyArg2Scalar(other_obj, "__sub__", 0); + { + eager_gil_scoped_release guard; + other_tensor = full_ad_func(self_tensor.shape(), + value, + self_tensor.dtype(), + self_tensor.place()); + } + } + const phi::distributed::ProcessMesh* mesh = nullptr; + if (InputsContainDistTensor(&mesh, self_tensor, other_tensor)) { + ConvertAllInputsToDistTensor(mesh, self_tensor, other_tensor); } - } - - const phi::distributed::ProcessMesh* mesh = nullptr; - if (InputsContainDistTensor(&mesh, self_tensor, other_tensor)) { - ConvertAllInputsToDistTensor(mesh, self_tensor, other_tensor); } // 3. promote types or unify right var type to left var, float type promotion @@ -444,24 +465,34 @@ static PyObject* tensor__rsub__method(TensorObject* self, // 2. create or get tensor for other_obj paddle::Tensor other_tensor; if (PyCheckTensor(other_obj)) { - other_tensor = CastPyArg2Tensor(other_obj, 0); - } else if (IsNumpyArray(other_obj)) { - py::object numpy_value = py::object(py::handle(other_obj), true); - other_tensor = paddle::Tensor(place); - InitTensorWithNumpyValue(numpy_value, place, &other_tensor); + auto& other_tensor_ref = CastPyArg2Tensor(other_obj, 0); + auto& self_tensor_ref = self->tensor; + const phi::distributed::ProcessMesh* mesh = nullptr; + if (InputsContainDistTensor(&mesh, self_tensor_ref, other_tensor_ref)) { + ConvertAllInputsToDistTensor(mesh, self_tensor_ref, other_tensor_ref); + } + self_tensor = self_tensor_ref; + other_tensor = other_tensor_ref; } else { - paddle::experimental::Scalar value = - CastPyArg2Scalar(other_obj, "__rsub__", 0); - { - eager_gil_scoped_release guard; - other_tensor = full_ad_func( - self_tensor.shape(), value, self_tensor.dtype(), self_tensor.place()); + if (IsNumpyArray(other_obj)) { + py::object numpy_value = py::object(py::handle(other_obj), true); + other_tensor = paddle::Tensor(place); + InitTensorWithNumpyValue(numpy_value, place, &other_tensor); + } else { + paddle::experimental::Scalar value = + CastPyArg2Scalar(other_obj, "__rsub__", 0); + { + eager_gil_scoped_release guard; + other_tensor = full_ad_func(self_tensor.shape(), + value, + self_tensor.dtype(), + self_tensor.place()); + } + } + const phi::distributed::ProcessMesh* mesh = nullptr; + if (InputsContainDistTensor(&mesh, self_tensor, other_tensor)) { + ConvertAllInputsToDistTensor(mesh, self_tensor, other_tensor); } - } - - const phi::distributed::ProcessMesh* mesh = nullptr; - if (InputsContainDistTensor(&mesh, self_tensor, other_tensor)) { - ConvertAllInputsToDistTensor(mesh, self_tensor, other_tensor); } // 3. promote types or unify right var type to left var, float type promotion @@ -521,8 +552,8 @@ static PyObject* tensor__mul__method(TensorObject* self, SetDevice(place); paddle::Tensor ret; - paddle::Tensor self_tensor = self->tensor; + paddle::Tensor self_tensor = self->tensor; PyObject* other_obj = PyTuple_GET_ITEM(args, 0); // 1. scalar exists cases @@ -547,32 +578,46 @@ static PyObject* tensor__mul__method(TensorObject* self, } // 2. create or get tensor for other_obj + // if lhs or rhs input is tensor, we need to inplace cast it to dist_tensor + // if one of the input is numpy or scalar, no need to do inplace cast. paddle::Tensor other_tensor; if (PyCheckTensor(other_obj)) { - other_tensor = CastPyArg2Tensor(other_obj, 0); - } else if (IsNumpyArray(other_obj)) { - py::object numpy_value = py::object(py::handle(other_obj), true); - other_tensor = paddle::Tensor(place); - InitTensorWithNumpyValue(numpy_value, place, &other_tensor); + auto& self_tensor_ref = self->tensor; + auto& other_tensor_ref = CastPyArg2Tensor(other_obj, 0); + const phi::distributed::ProcessMesh* mesh = nullptr; + if (InputsContainDistTensor(&mesh, self_tensor_ref, other_tensor_ref)) { + ConvertAllInputsToDistTensor(mesh, self_tensor_ref, other_tensor_ref); + } + self_tensor = self_tensor_ref; + other_tensor = other_tensor_ref; } else { - paddle::experimental::Scalar value = - CastPyArg2Scalar(other_obj, "__mul__", 0); - if (PyComplex_Check(other_obj)) { - eager_gil_scoped_release guard; - other_tensor = full_ad_func( - self_tensor.shape(), value, DataType::COMPLEX64, self_tensor.place()); + if (IsNumpyArray(other_obj)) { + py::object numpy_value = py::object(py::handle(other_obj), true); + other_tensor = paddle::Tensor(place); + InitTensorWithNumpyValue(numpy_value, place, &other_tensor); } else { - eager_gil_scoped_release guard; - other_tensor = full_ad_func( - self_tensor.shape(), value, self_tensor.dtype(), self_tensor.place()); + paddle::experimental::Scalar value = + CastPyArg2Scalar(other_obj, "__mul__", 0); + if (PyComplex_Check(other_obj)) { + eager_gil_scoped_release guard; + other_tensor = full_ad_func(self_tensor.shape(), + value, + DataType::COMPLEX64, + self_tensor.place()); + } else { + eager_gil_scoped_release guard; + other_tensor = full_ad_func(self_tensor.shape(), + value, + self_tensor.dtype(), + self_tensor.place()); + } + const phi::distributed::ProcessMesh* mesh = nullptr; + if (InputsContainDistTensor(&mesh, self_tensor, other_tensor)) { + ConvertAllInputsToDistTensor(mesh, self_tensor, other_tensor); + } } } - const phi::distributed::ProcessMesh* mesh = nullptr; - if (InputsContainDistTensor(&mesh, self_tensor, other_tensor)) { - ConvertAllInputsToDistTensor(mesh, self_tensor, other_tensor); - } - // 3. promote types or unify right var type to left var, float type promotion // mv to multiply_ad_func phi::DataType lhs_dtype = self_tensor.dtype(); @@ -633,8 +678,8 @@ static PyObject* tensor__div__method(TensorObject* self, SetDevice(place); paddle::Tensor ret; - paddle::Tensor self_tensor = self->tensor; + paddle::Tensor self_tensor = self->tensor; PyObject* other_obj = PyTuple_GET_ITEM(args, 0); // 1. scalar exists cases @@ -661,28 +706,38 @@ static PyObject* tensor__div__method(TensorObject* self, // 2. create or get tensor for other_obj paddle::Tensor other_tensor; if (PyCheckTensor(other_obj)) { - other_tensor = CastPyArg2Tensor(other_obj, 0); - } else if (IsNumpyArray(other_obj)) { - py::object numpy_value = py::object(py::handle(other_obj), true); - other_tensor = paddle::Tensor(place); - InitTensorWithNumpyValue(numpy_value, place, &other_tensor); + auto& self_tensor_ref = self->tensor; + auto& other_tensor_ref = CastPyArg2Tensor(other_obj, 0); + const phi::distributed::ProcessMesh* mesh = nullptr; + if (InputsContainDistTensor(&mesh, self_tensor_ref, other_tensor_ref)) { + ConvertAllInputsToDistTensor(mesh, self_tensor_ref, other_tensor_ref); + } + self_tensor = self_tensor_ref; + other_tensor = other_tensor_ref; } else { - paddle::experimental::Scalar value = - CastPyArg2Scalar(other_obj, "__div__", 0); - if (PyComplex_Check(other_obj)) { - eager_gil_scoped_release guard; - other_tensor = - full_ad_func({1}, value, DataType::COMPLEX64, self_tensor.place()); + if (IsNumpyArray(other_obj)) { + py::object numpy_value = py::object(py::handle(other_obj), true); + other_tensor = paddle::Tensor(place); + InitTensorWithNumpyValue(numpy_value, place, &other_tensor); } else { - eager_gil_scoped_release guard; - other_tensor = full_ad_func( - self_tensor.shape(), value, self_tensor.dtype(), self_tensor.place()); + paddle::experimental::Scalar value = + CastPyArg2Scalar(other_obj, "__div__", 0); + if (PyComplex_Check(other_obj)) { + eager_gil_scoped_release guard; + other_tensor = + full_ad_func({1}, value, DataType::COMPLEX64, self_tensor.place()); + } else { + eager_gil_scoped_release guard; + other_tensor = full_ad_func(self_tensor.shape(), + value, + self_tensor.dtype(), + self_tensor.place()); + } + } + const phi::distributed::ProcessMesh* mesh = nullptr; + if (InputsContainDistTensor(&mesh, self_tensor, other_tensor)) { + ConvertAllInputsToDistTensor(mesh, self_tensor, other_tensor); } - } - - const phi::distributed::ProcessMesh* mesh = nullptr; - if (InputsContainDistTensor(&mesh, self_tensor, other_tensor)) { - ConvertAllInputsToDistTensor(mesh, self_tensor, other_tensor); } // 3. promote types or unify right var type to left var @@ -753,8 +808,8 @@ static PyObject* tensor__rdiv__method(TensorObject* self, SetDevice(place); paddle::Tensor ret; - paddle::Tensor self_tensor = self->tensor; + paddle::Tensor self_tensor = self->tensor; PyObject* other_obj = PyTuple_GET_ITEM(args, 0); // 1. scalar exists cases @@ -786,28 +841,38 @@ static PyObject* tensor__rdiv__method(TensorObject* self, self_tensor.dtype(), place); } else if (PyCheckTensor(other_obj)) { - other_tensor = CastPyArg2Tensor(other_obj, 0); - } else if (IsNumpyArray(other_obj)) { - py::object numpy_value = py::object(py::handle(other_obj), true); - other_tensor = paddle::Tensor(place); - InitTensorWithNumpyValue(numpy_value, place, &other_tensor); + auto& self_tensor_ref = self->tensor; + auto& other_tensor_ref = CastPyArg2Tensor(other_obj, 0); + const phi::distributed::ProcessMesh* mesh = nullptr; + if (InputsContainDistTensor(&mesh, self_tensor_ref, other_tensor_ref)) { + ConvertAllInputsToDistTensor(mesh, self_tensor_ref, other_tensor_ref); + } + self_tensor = self_tensor_ref; + other_tensor = other_tensor_ref; } else { - paddle::experimental::Scalar value = - CastPyArg2Scalar(other_obj, "__rdiv__", 0); - if (PyComplex_Check(other_obj)) { - eager_gil_scoped_release guard; - other_tensor = - full_ad_func({1}, value, DataType::COMPLEX64, self_tensor.place()); + if (IsNumpyArray(other_obj)) { + py::object numpy_value = py::object(py::handle(other_obj), true); + other_tensor = paddle::Tensor(place); + InitTensorWithNumpyValue(numpy_value, place, &other_tensor); } else { - eager_gil_scoped_release guard; - other_tensor = full_ad_func( - self_tensor.shape(), value, self_tensor.dtype(), self_tensor.place()); + paddle::experimental::Scalar value = + CastPyArg2Scalar(other_obj, "__rdiv__", 0); + if (PyComplex_Check(other_obj)) { + eager_gil_scoped_release guard; + other_tensor = + full_ad_func({1}, value, DataType::COMPLEX64, self_tensor.place()); + } else { + eager_gil_scoped_release guard; + other_tensor = full_ad_func(self_tensor.shape(), + value, + self_tensor.dtype(), + self_tensor.place()); + } + } + const phi::distributed::ProcessMesh* mesh = nullptr; + if (InputsContainDistTensor(&mesh, self_tensor, other_tensor)) { + ConvertAllInputsToDistTensor(mesh, self_tensor, other_tensor); } - } - - const phi::distributed::ProcessMesh* mesh = nullptr; - if (InputsContainDistTensor(&mesh, self_tensor, other_tensor)) { - ConvertAllInputsToDistTensor(mesh, self_tensor, other_tensor); } // 3. promote types or unify right var type to left var @@ -909,28 +974,38 @@ static PyObject* tensor__gt__method(TensorObject* self, self_tensor.dtype(), place); } else if (PyCheckTensor(other_obj)) { - other_tensor = CastPyArg2Tensor(other_obj, 0); - } else if (IsNumpyArray(other_obj)) { - py::object numpy_value = py::object(py::handle(other_obj), true); - other_tensor = paddle::Tensor(place); - InitTensorWithNumpyValue(numpy_value, place, &other_tensor); + auto& self_tensor_ref = self->tensor; + auto& other_tensor_ref = CastPyArg2Tensor(other_obj, 0); + const phi::distributed::ProcessMesh* mesh = nullptr; + if (InputsContainDistTensor(&mesh, self_tensor_ref, other_tensor_ref)) { + ConvertAllInputsToDistTensor(mesh, self_tensor_ref, other_tensor_ref); + } + self_tensor = self_tensor_ref; + other_tensor = other_tensor_ref; } else { - paddle::experimental::Scalar value = - CastPyArg2Scalar(other_obj, "__gt__", 0); - if (PyComplex_Check(other_obj)) { - eager_gil_scoped_release guard; - other_tensor = - full_ad_func({1}, value, DataType::COMPLEX64, self_tensor.place()); + if (IsNumpyArray(other_obj)) { + py::object numpy_value = py::object(py::handle(other_obj), true); + other_tensor = paddle::Tensor(place); + InitTensorWithNumpyValue(numpy_value, place, &other_tensor); } else { - eager_gil_scoped_release guard; - other_tensor = full_ad_func( - self_tensor.shape(), value, self_tensor.dtype(), self_tensor.place()); + paddle::experimental::Scalar value = + CastPyArg2Scalar(other_obj, "__gt__", 0); + if (PyComplex_Check(other_obj)) { + eager_gil_scoped_release guard; + other_tensor = + full_ad_func({1}, value, DataType::COMPLEX64, self_tensor.place()); + } else { + eager_gil_scoped_release guard; + other_tensor = full_ad_func(self_tensor.shape(), + value, + self_tensor.dtype(), + self_tensor.place()); + } + } + const phi::distributed::ProcessMesh* mesh = nullptr; + if (InputsContainDistTensor(&mesh, self_tensor, other_tensor)) { + ConvertAllInputsToDistTensor(mesh, self_tensor, other_tensor); } - } - - const phi::distributed::ProcessMesh* mesh = nullptr; - if (InputsContainDistTensor(&mesh, self_tensor, other_tensor)) { - ConvertAllInputsToDistTensor(mesh, self_tensor, other_tensor); } // 3. promote types or unify right var type to left var @@ -1004,28 +1079,38 @@ static PyObject* tensor__ge__method(TensorObject* self, self_tensor.dtype(), place); } else if (PyCheckTensor(other_obj)) { - other_tensor = CastPyArg2Tensor(other_obj, 0); - } else if (IsNumpyArray(other_obj)) { - py::object numpy_value = py::object(py::handle(other_obj), true); - other_tensor = paddle::Tensor(place); - InitTensorWithNumpyValue(numpy_value, place, &other_tensor); + auto& self_tensor_ref = self->tensor; + auto& other_tensor_ref = CastPyArg2Tensor(other_obj, 0); + const phi::distributed::ProcessMesh* mesh = nullptr; + if (InputsContainDistTensor(&mesh, self_tensor_ref, other_tensor_ref)) { + ConvertAllInputsToDistTensor(mesh, self_tensor_ref, other_tensor_ref); + } + self_tensor = self_tensor_ref; + other_tensor = other_tensor_ref; } else { - paddle::experimental::Scalar value = - CastPyArg2Scalar(other_obj, "__ge__", 0); - if (PyComplex_Check(other_obj)) { - eager_gil_scoped_release guard; - other_tensor = - full_ad_func({1}, value, DataType::COMPLEX64, self_tensor.place()); + if (IsNumpyArray(other_obj)) { + py::object numpy_value = py::object(py::handle(other_obj), true); + other_tensor = paddle::Tensor(place); + InitTensorWithNumpyValue(numpy_value, place, &other_tensor); } else { - eager_gil_scoped_release guard; - other_tensor = full_ad_func( - self_tensor.shape(), value, self_tensor.dtype(), self_tensor.place()); + paddle::experimental::Scalar value = + CastPyArg2Scalar(other_obj, "__ge__", 0); + if (PyComplex_Check(other_obj)) { + eager_gil_scoped_release guard; + other_tensor = + full_ad_func({1}, value, DataType::COMPLEX64, self_tensor.place()); + } else { + eager_gil_scoped_release guard; + other_tensor = full_ad_func(self_tensor.shape(), + value, + self_tensor.dtype(), + self_tensor.place()); + } + } + const phi::distributed::ProcessMesh* mesh = nullptr; + if (InputsContainDistTensor(&mesh, self_tensor, other_tensor)) { + ConvertAllInputsToDistTensor(mesh, self_tensor, other_tensor); } - } - - const phi::distributed::ProcessMesh* mesh = nullptr; - if (InputsContainDistTensor(&mesh, self_tensor, other_tensor)) { - ConvertAllInputsToDistTensor(mesh, self_tensor, other_tensor); } // 3. promote types or unify right var type to left var @@ -1067,8 +1152,8 @@ static PyObject* tensor__mod__method(TensorObject* self, SetDevice(place); paddle::Tensor ret; - paddle::Tensor self_tensor = self->tensor; + paddle::Tensor self_tensor = self->tensor; PyObject* other_obj = PyTuple_GET_ITEM(args, 0); // 1. scalar exists cases @@ -1100,28 +1185,38 @@ static PyObject* tensor__mod__method(TensorObject* self, self_tensor.dtype(), self_tensor.place()); } else if (PyCheckTensor(other_obj)) { - other_tensor = CastPyArg2Tensor(other_obj, 0); - } else if (IsNumpyArray(other_obj)) { - py::object numpy_value = py::object(py::handle(other_obj), true); - other_tensor = paddle::Tensor(place); - InitTensorWithNumpyValue(numpy_value, place, &other_tensor); + auto& self_tensor_ref = self->tensor; + auto& other_tensor_ref = CastPyArg2Tensor(other_obj, 0); + const phi::distributed::ProcessMesh* mesh = nullptr; + if (InputsContainDistTensor(&mesh, self_tensor_ref, other_tensor_ref)) { + ConvertAllInputsToDistTensor(mesh, self_tensor_ref, other_tensor_ref); + } + self_tensor = self_tensor_ref; + other_tensor = other_tensor_ref; } else { - paddle::experimental::Scalar value = - CastPyArg2Scalar(other_obj, "__mod__", 0); - if (PyComplex_Check(other_obj)) { - eager_gil_scoped_release guard; - other_tensor = - full_ad_func({1}, value, DataType::COMPLEX64, self_tensor.place()); + if (IsNumpyArray(other_obj)) { + py::object numpy_value = py::object(py::handle(other_obj), true); + other_tensor = paddle::Tensor(place); + InitTensorWithNumpyValue(numpy_value, place, &other_tensor); } else { - eager_gil_scoped_release guard; - other_tensor = full_ad_func( - self_tensor.shape(), value, self_tensor.dtype(), self_tensor.place()); + paddle::experimental::Scalar value = + CastPyArg2Scalar(other_obj, "__mod__", 0); + if (PyComplex_Check(other_obj)) { + eager_gil_scoped_release guard; + other_tensor = + full_ad_func({1}, value, DataType::COMPLEX64, self_tensor.place()); + } else { + eager_gil_scoped_release guard; + other_tensor = full_ad_func(self_tensor.shape(), + value, + self_tensor.dtype(), + self_tensor.place()); + } + } + const phi::distributed::ProcessMesh* mesh = nullptr; + if (InputsContainDistTensor(&mesh, self_tensor, other_tensor)) { + ConvertAllInputsToDistTensor(mesh, self_tensor, other_tensor); } - } - - const phi::distributed::ProcessMesh* mesh = nullptr; - if (InputsContainDistTensor(&mesh, self_tensor, other_tensor)) { - ConvertAllInputsToDistTensor(mesh, self_tensor, other_tensor); } // 3. promote types or unify right var type to left var @@ -1195,28 +1290,36 @@ static PyObject* tensor__matmul__method(TensorObject* self, self_tensor.dtype(), self_tensor.place()); } else if (PyCheckTensor(other_obj)) { - other_tensor = CastPyArg2Tensor(other_obj, 0); - } else if (IsNumpyArray(other_obj)) { - py::object numpy_value = py::object(py::handle(other_obj), true); - other_tensor = paddle::Tensor(place); - InitTensorWithNumpyValue(numpy_value, place, &other_tensor); + auto& self_tensor_ref = self->tensor; + auto& other_tensor_ref = CastPyArg2Tensor(other_obj, 0); + const phi::distributed::ProcessMesh* mesh = nullptr; + if (InputsContainDistTensor(&mesh, self_tensor_ref, other_tensor_ref)) { + ConvertAllInputsToDistTensor(mesh, self_tensor_ref, other_tensor_ref); + } + self_tensor = self_tensor_ref; + other_tensor = other_tensor_ref; } else { - paddle::experimental::Scalar value = - CastPyArg2Scalar(other_obj, "__matmul__", 0); - if (PyComplex_Check(other_obj)) { - eager_gil_scoped_release guard; - other_tensor = - full_ad_func({1}, value, DataType::COMPLEX64, self_tensor.place()); + if (IsNumpyArray(other_obj)) { + py::object numpy_value = py::object(py::handle(other_obj), true); + other_tensor = paddle::Tensor(place); + InitTensorWithNumpyValue(numpy_value, place, &other_tensor); } else { - eager_gil_scoped_release guard; - other_tensor = - full_ad_func({1}, value, self_tensor.dtype(), self_tensor.place()); + paddle::experimental::Scalar value = + CastPyArg2Scalar(other_obj, "__matmul__", 0); + if (PyComplex_Check(other_obj)) { + eager_gil_scoped_release guard; + other_tensor = + full_ad_func({1}, value, DataType::COMPLEX64, self_tensor.place()); + } else { + eager_gil_scoped_release guard; + other_tensor = + full_ad_func({1}, value, self_tensor.dtype(), self_tensor.place()); + } + } + const phi::distributed::ProcessMesh* mesh = nullptr; + if (InputsContainDistTensor(&mesh, self_tensor, other_tensor)) { + ConvertAllInputsToDistTensor(mesh, self_tensor, other_tensor); } - } - - const phi::distributed::ProcessMesh* mesh = nullptr; - if (InputsContainDistTensor(&mesh, self_tensor, other_tensor)) { - ConvertAllInputsToDistTensor(mesh, self_tensor, other_tensor); } // 3. promote types or unify right var type to left var @@ -1308,28 +1411,38 @@ static PyObject* tensor__lt__method(TensorObject* self, self_tensor.dtype(), self_tensor.place()); } else if (PyCheckTensor(other_obj)) { - other_tensor = CastPyArg2Tensor(other_obj, 0); - } else if (IsNumpyArray(other_obj)) { - py::object numpy_value = py::object(py::handle(other_obj), true); - other_tensor = paddle::Tensor(place); - InitTensorWithNumpyValue(numpy_value, place, &other_tensor); + auto& self_tensor_ref = self->tensor; + auto& other_tensor_ref = CastPyArg2Tensor(other_obj, 0); + const phi::distributed::ProcessMesh* mesh = nullptr; + if (InputsContainDistTensor(&mesh, self_tensor_ref, other_tensor_ref)) { + ConvertAllInputsToDistTensor(mesh, self_tensor_ref, other_tensor_ref); + } + self_tensor = self_tensor_ref; + other_tensor = other_tensor_ref; } else { - paddle::experimental::Scalar value = - CastPyArg2Scalar(other_obj, "__lt__", 0); - if (PyComplex_Check(other_obj)) { - eager_gil_scoped_release guard; - other_tensor = - full_ad_func({1}, value, DataType::COMPLEX64, self_tensor.place()); + if (IsNumpyArray(other_obj)) { + py::object numpy_value = py::object(py::handle(other_obj), true); + other_tensor = paddle::Tensor(place); + InitTensorWithNumpyValue(numpy_value, place, &other_tensor); } else { - eager_gil_scoped_release guard; - other_tensor = full_ad_func( - self_tensor.shape(), value, self_tensor.dtype(), self_tensor.place()); + paddle::experimental::Scalar value = + CastPyArg2Scalar(other_obj, "__lt__", 0); + if (PyComplex_Check(other_obj)) { + eager_gil_scoped_release guard; + other_tensor = + full_ad_func({1}, value, DataType::COMPLEX64, self_tensor.place()); + } else { + eager_gil_scoped_release guard; + other_tensor = full_ad_func(self_tensor.shape(), + value, + self_tensor.dtype(), + self_tensor.place()); + } + } + const phi::distributed::ProcessMesh* mesh = nullptr; + if (InputsContainDistTensor(&mesh, self_tensor, other_tensor)) { + ConvertAllInputsToDistTensor(mesh, self_tensor, other_tensor); } - } - - const phi::distributed::ProcessMesh* mesh = nullptr; - if (InputsContainDistTensor(&mesh, self_tensor, other_tensor)) { - ConvertAllInputsToDistTensor(mesh, self_tensor, other_tensor); } // 3. promote types or unify right var type to left var @@ -1403,28 +1516,38 @@ static PyObject* tensor__le__method(TensorObject* self, self_tensor.dtype(), self_tensor.place()); } else if (PyCheckTensor(other_obj)) { - other_tensor = CastPyArg2Tensor(other_obj, 0); - } else if (IsNumpyArray(other_obj)) { - py::object numpy_value = py::object(py::handle(other_obj), true); - other_tensor = paddle::Tensor(place); - InitTensorWithNumpyValue(numpy_value, place, &other_tensor); + auto& self_tensor_ref = self->tensor; + auto& other_tensor_ref = CastPyArg2Tensor(other_obj, 0); + const phi::distributed::ProcessMesh* mesh = nullptr; + if (InputsContainDistTensor(&mesh, self_tensor_ref, other_tensor_ref)) { + ConvertAllInputsToDistTensor(mesh, self_tensor_ref, other_tensor_ref); + } + self_tensor = self_tensor_ref; + other_tensor = other_tensor_ref; } else { - paddle::experimental::Scalar value = - CastPyArg2Scalar(other_obj, "__le__", 0); - if (PyComplex_Check(other_obj)) { - eager_gil_scoped_release guard; - other_tensor = - full_ad_func({1}, value, DataType::COMPLEX64, self_tensor.place()); + if (IsNumpyArray(other_obj)) { + py::object numpy_value = py::object(py::handle(other_obj), true); + other_tensor = paddle::Tensor(place); + InitTensorWithNumpyValue(numpy_value, place, &other_tensor); } else { - eager_gil_scoped_release guard; - other_tensor = full_ad_func( - self_tensor.shape(), value, self_tensor.dtype(), self_tensor.place()); + paddle::experimental::Scalar value = + CastPyArg2Scalar(other_obj, "__le__", 0); + if (PyComplex_Check(other_obj)) { + eager_gil_scoped_release guard; + other_tensor = + full_ad_func({1}, value, DataType::COMPLEX64, self_tensor.place()); + } else { + eager_gil_scoped_release guard; + other_tensor = full_ad_func(self_tensor.shape(), + value, + self_tensor.dtype(), + self_tensor.place()); + } + } + const phi::distributed::ProcessMesh* mesh = nullptr; + if (InputsContainDistTensor(&mesh, self_tensor, other_tensor)) { + ConvertAllInputsToDistTensor(mesh, self_tensor, other_tensor); } - } - - const phi::distributed::ProcessMesh* mesh = nullptr; - if (InputsContainDistTensor(&mesh, self_tensor, other_tensor)) { - ConvertAllInputsToDistTensor(mesh, self_tensor, other_tensor); } // 3. promote types or unify right var type to left var @@ -1499,28 +1622,36 @@ static PyObject* tensor__floordiv__method(TensorObject* self, self_tensor.dtype(), self_tensor.place()); } else if (PyCheckTensor(other_obj)) { - other_tensor = CastPyArg2Tensor(other_obj, 0); - } else if (IsNumpyArray(other_obj)) { - py::object numpy_value = py::object(py::handle(other_obj), true); - other_tensor = paddle::Tensor(place); - InitTensorWithNumpyValue(numpy_value, place, &other_tensor); + auto& self_tensor_ref = self->tensor; + auto& other_tensor_ref = CastPyArg2Tensor(other_obj, 0); + const phi::distributed::ProcessMesh* mesh = nullptr; + if (InputsContainDistTensor(&mesh, self_tensor_ref, other_tensor_ref)) { + ConvertAllInputsToDistTensor(mesh, self_tensor_ref, other_tensor_ref); + } + self_tensor = self_tensor_ref; + other_tensor = other_tensor_ref; } else { - paddle::experimental::Scalar value = - CastPyArg2Scalar(other_obj, "__floordiv__", 0); - if (PyComplex_Check(other_obj)) { - eager_gil_scoped_release guard; - other_tensor = - full_ad_func({1}, value, DataType::COMPLEX64, self_tensor.place()); + if (IsNumpyArray(other_obj)) { + py::object numpy_value = py::object(py::handle(other_obj), true); + other_tensor = paddle::Tensor(place); + InitTensorWithNumpyValue(numpy_value, place, &other_tensor); } else { - eager_gil_scoped_release guard; - other_tensor = - full_ad_func({1}, value, self_tensor.dtype(), self_tensor.place()); + paddle::experimental::Scalar value = + CastPyArg2Scalar(other_obj, "__floordiv__", 0); + if (PyComplex_Check(other_obj)) { + eager_gil_scoped_release guard; + other_tensor = + full_ad_func({1}, value, DataType::COMPLEX64, self_tensor.place()); + } else { + eager_gil_scoped_release guard; + other_tensor = + full_ad_func({1}, value, self_tensor.dtype(), self_tensor.place()); + } + } + const phi::distributed::ProcessMesh* mesh = nullptr; + if (InputsContainDistTensor(&mesh, self_tensor, other_tensor)) { + ConvertAllInputsToDistTensor(mesh, self_tensor, other_tensor); } - } - - const phi::distributed::ProcessMesh* mesh = nullptr; - if (InputsContainDistTensor(&mesh, self_tensor, other_tensor)) { - ConvertAllInputsToDistTensor(mesh, self_tensor, other_tensor); } // 3. promote types or unify right var type to left var @@ -1593,28 +1724,36 @@ static PyObject* tensor__pow__method(TensorObject* self, // 2. create or get tensor for other_obj paddle::Tensor other_tensor; if (PyCheckTensor(other_obj)) { - other_tensor = CastPyArg2Tensor(other_obj, 0); - } else if (IsNumpyArray(other_obj)) { - py::object numpy_value = py::object(py::handle(other_obj), true); - other_tensor = paddle::Tensor(place); - InitTensorWithNumpyValue(numpy_value, place, &other_tensor); + auto& self_tensor_ref = self->tensor; + auto& other_tensor_ref = CastPyArg2Tensor(other_obj, 0); + const phi::distributed::ProcessMesh* mesh = nullptr; + if (InputsContainDistTensor(&mesh, self_tensor_ref, other_tensor_ref)) { + ConvertAllInputsToDistTensor(mesh, self_tensor_ref, other_tensor_ref); + } + self_tensor = self_tensor_ref; + other_tensor = other_tensor_ref; } else { - paddle::experimental::Scalar value = - CastPyArg2Scalar(other_obj, "__pow__", 0); - if (PyComplex_Check(other_obj)) { - eager_gil_scoped_release guard; - other_tensor = - full_ad_func({1}, value, DataType::COMPLEX64, self_tensor.place()); + if (IsNumpyArray(other_obj)) { + py::object numpy_value = py::object(py::handle(other_obj), true); + other_tensor = paddle::Tensor(place); + InitTensorWithNumpyValue(numpy_value, place, &other_tensor); } else { - eager_gil_scoped_release guard; - other_tensor = - full_ad_func({1}, value, self_tensor.dtype(), self_tensor.place()); + paddle::experimental::Scalar value = + CastPyArg2Scalar(other_obj, "__pow__", 0); + if (PyComplex_Check(other_obj)) { + eager_gil_scoped_release guard; + other_tensor = + full_ad_func({1}, value, DataType::COMPLEX64, self_tensor.place()); + } else { + eager_gil_scoped_release guard; + other_tensor = + full_ad_func({1}, value, self_tensor.dtype(), self_tensor.place()); + } + } + const phi::distributed::ProcessMesh* mesh = nullptr; + if (InputsContainDistTensor(&mesh, self_tensor, other_tensor)) { + ConvertAllInputsToDistTensor(mesh, self_tensor, other_tensor); } - } - - const phi::distributed::ProcessMesh* mesh = nullptr; - if (InputsContainDistTensor(&mesh, self_tensor, other_tensor)) { - ConvertAllInputsToDistTensor(mesh, self_tensor, other_tensor); } // 3. promote types or unify right var type to left var @@ -1690,28 +1829,38 @@ static PyObject* tensor__rpow__method(TensorObject* self, self_tensor.dtype(), self_tensor.place()); } else if (PyCheckTensor(other_obj)) { - other_tensor = CastPyArg2Tensor(other_obj, 0); - } else if (IsNumpyArray(other_obj)) { - py::object numpy_value = py::object(py::handle(other_obj), true); - other_tensor = paddle::Tensor(place); - InitTensorWithNumpyValue(numpy_value, place, &other_tensor); + auto& self_tensor_ref = self->tensor; + auto& other_tensor_ref = CastPyArg2Tensor(other_obj, 0); + const phi::distributed::ProcessMesh* mesh = nullptr; + if (InputsContainDistTensor(&mesh, self_tensor_ref, other_tensor_ref)) { + ConvertAllInputsToDistTensor(mesh, self_tensor_ref, other_tensor_ref); + } + self_tensor = self_tensor_ref; + other_tensor = other_tensor_ref; } else { - paddle::experimental::Scalar value = - CastPyArg2Scalar(other_obj, "__rpow__", 0); - if (PyComplex_Check(other_obj)) { - eager_gil_scoped_release guard; - other_tensor = - full_ad_func({1}, value, DataType::COMPLEX64, self_tensor.place()); + if (IsNumpyArray(other_obj)) { + py::object numpy_value = py::object(py::handle(other_obj), true); + other_tensor = paddle::Tensor(place); + InitTensorWithNumpyValue(numpy_value, place, &other_tensor); } else { - eager_gil_scoped_release guard; - other_tensor = full_ad_func( - self_tensor.shape(), value, self_tensor.dtype(), self_tensor.place()); + paddle::experimental::Scalar value = + CastPyArg2Scalar(other_obj, "__rpow__", 0); + if (PyComplex_Check(other_obj)) { + eager_gil_scoped_release guard; + other_tensor = + full_ad_func({1}, value, DataType::COMPLEX64, self_tensor.place()); + } else { + eager_gil_scoped_release guard; + other_tensor = full_ad_func(self_tensor.shape(), + value, + self_tensor.dtype(), + self_tensor.place()); + } + } + const phi::distributed::ProcessMesh* mesh = nullptr; + if (InputsContainDistTensor(&mesh, self_tensor, other_tensor)) { + ConvertAllInputsToDistTensor(mesh, self_tensor, other_tensor); } - } - - const phi::distributed::ProcessMesh* mesh = nullptr; - if (InputsContainDistTensor(&mesh, self_tensor, other_tensor)) { - ConvertAllInputsToDistTensor(mesh, self_tensor, other_tensor); } // 3. promote types or unify right var type to left var @@ -1785,28 +1934,36 @@ static PyObject* tensor__ne__method(TensorObject* self, self_tensor.dtype(), self_tensor.place()); } else if (PyCheckTensor(other_obj)) { - other_tensor = CastPyArg2Tensor(other_obj, 0); - } else if (IsNumpyArray(other_obj)) { - py::object numpy_value = py::object(py::handle(other_obj), true); - other_tensor = paddle::Tensor(place); - InitTensorWithNumpyValue(numpy_value, place, &other_tensor); + auto& self_tensor_ref = self->tensor; + auto& other_tensor_ref = CastPyArg2Tensor(other_obj, 0); + const phi::distributed::ProcessMesh* mesh = nullptr; + if (InputsContainDistTensor(&mesh, self_tensor_ref, other_tensor_ref)) { + ConvertAllInputsToDistTensor(mesh, self_tensor_ref, other_tensor_ref); + } + self_tensor = self_tensor_ref; + other_tensor = other_tensor_ref; } else { - paddle::experimental::Scalar value = - CastPyArg2Scalar(other_obj, "__ne__", 0); - if (PyComplex_Check(other_obj)) { - eager_gil_scoped_release guard; - other_tensor = - full_ad_func({1}, value, DataType::COMPLEX64, self_tensor.place()); + if (IsNumpyArray(other_obj)) { + py::object numpy_value = py::object(py::handle(other_obj), true); + other_tensor = paddle::Tensor(place); + InitTensorWithNumpyValue(numpy_value, place, &other_tensor); } else { - eager_gil_scoped_release guard; - other_tensor = - full_ad_func({1}, value, self_tensor.dtype(), self_tensor.place()); + paddle::experimental::Scalar value = + CastPyArg2Scalar(other_obj, "__ne__", 0); + if (PyComplex_Check(other_obj)) { + eager_gil_scoped_release guard; + other_tensor = + full_ad_func({1}, value, DataType::COMPLEX64, self_tensor.place()); + } else { + eager_gil_scoped_release guard; + other_tensor = + full_ad_func({1}, value, self_tensor.dtype(), self_tensor.place()); + } + } + const phi::distributed::ProcessMesh* mesh = nullptr; + if (InputsContainDistTensor(&mesh, self_tensor, other_tensor)) { + ConvertAllInputsToDistTensor(mesh, self_tensor, other_tensor); } - } - - const phi::distributed::ProcessMesh* mesh = nullptr; - if (InputsContainDistTensor(&mesh, self_tensor, other_tensor)) { - ConvertAllInputsToDistTensor(mesh, self_tensor, other_tensor); } // 3. promote types or unify right var type to left var @@ -1880,28 +2037,36 @@ static PyObject* tensor__eq__method(TensorObject* self, self_tensor.dtype(), self_tensor.place()); } else if (PyCheckTensor(other_obj)) { - other_tensor = CastPyArg2Tensor(other_obj, 0); - } else if (IsNumpyArray(other_obj)) { - py::object numpy_value = py::object(py::handle(other_obj), true); - other_tensor = paddle::Tensor(place); - InitTensorWithNumpyValue(numpy_value, place, &other_tensor); + auto& self_tensor_ref = self->tensor; + auto& other_tensor_ref = CastPyArg2Tensor(other_obj, 0); + const phi::distributed::ProcessMesh* mesh = nullptr; + if (InputsContainDistTensor(&mesh, self_tensor_ref, other_tensor_ref)) { + ConvertAllInputsToDistTensor(mesh, self_tensor_ref, other_tensor_ref); + } + self_tensor = self_tensor_ref; + other_tensor = other_tensor_ref; } else { - paddle::experimental::Scalar value = - CastPyArg2Scalar(other_obj, "__eq__", 0); - if (PyComplex_Check(other_obj)) { - eager_gil_scoped_release guard; - other_tensor = - full_ad_func({1}, value, DataType::COMPLEX64, self_tensor.place()); + if (IsNumpyArray(other_obj)) { + py::object numpy_value = py::object(py::handle(other_obj), true); + other_tensor = paddle::Tensor(place); + InitTensorWithNumpyValue(numpy_value, place, &other_tensor); } else { - eager_gil_scoped_release guard; - other_tensor = - full_ad_func({1}, value, self_tensor.dtype(), self_tensor.place()); + paddle::experimental::Scalar value = + CastPyArg2Scalar(other_obj, "__eq__", 0); + if (PyComplex_Check(other_obj)) { + eager_gil_scoped_release guard; + other_tensor = + full_ad_func({1}, value, DataType::COMPLEX64, self_tensor.place()); + } else { + eager_gil_scoped_release guard; + other_tensor = + full_ad_func({1}, value, self_tensor.dtype(), self_tensor.place()); + } + } + const phi::distributed::ProcessMesh* mesh = nullptr; + if (InputsContainDistTensor(&mesh, self_tensor, other_tensor)) { + ConvertAllInputsToDistTensor(mesh, self_tensor, other_tensor); } - } - - const phi::distributed::ProcessMesh* mesh = nullptr; - if (InputsContainDistTensor(&mesh, self_tensor, other_tensor)) { - ConvertAllInputsToDistTensor(mesh, self_tensor, other_tensor); } // 3. promote types or unify right var type to left var diff --git a/python/paddle/optimizer/optimizer.py b/python/paddle/optimizer/optimizer.py index 134b164409a95f..d4047b76b6a683 100644 --- a/python/paddle/optimizer/optimizer.py +++ b/python/paddle/optimizer/optimizer.py @@ -1174,7 +1174,7 @@ def _create_optimization_pass( # need to filter again here. if ( param_and_grad[1] is None - or not param_and_grad[1]._is_initialized() + or not param_and_grad[0]._is_initialized() ): continue if param_and_grad[0].stop_gradient is False: @@ -1185,7 +1185,7 @@ def _create_optimization_pass( for param_and_grad in parameters_and_grads['params']: if ( param_and_grad[1] is None - or not param_and_grad[1]._is_initialized() + or not param_and_grad[0]._is_initialized() ): continue if param_and_grad[0].stop_gradient is False: