diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index ed38235f..57a10dfe 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -56,7 +56,7 @@ jobs: strategy: fail-fast: false matrix: - backends: [ V8, JavaScriptCore, Lua, Empty ] + backends: [ V8, JavaScriptCore, Lua, Python, Empty ] build_type: - Debug - Release diff --git a/CMakeLists.txt b/CMakeLists.txt index eaa00368..20b431c3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,7 +39,7 @@ set(SCRIPTX_BACKEND_LIST ) # set options, choose which ScriptX Backend to use, V8 or JSC or etc... -set(SCRIPTX_BACKEND "" CACHE STRING "choose which ScriptX Backend(Implementation) to use, V8 or JavaScriptCore or etc...") +set(SCRIPTX_BACKEND "${SCRIPTX_BACKEND}" CACHE STRING "choose which ScriptX Backend(Implementation) to use, V8 or JavaScriptCore or etc...") set_property(CACHE SCRIPTX_BACKEND PROPERTY STRINGS "${SCRIPTX_BACKEND_LIST}") option(SCRIPTX_NO_EXCEPTION_ON_BIND_FUNCTION "don't throw exception on defineClass generated bound function/get/set, return null & log instead. default to OFF" OFF) option(SCRIPTX_FEATURE_INSPECTOR "enable inspector feature, default to OFF" OFF) @@ -177,4 +177,4 @@ message(STATUS "Configuring ScriptX using backend ${SCRIPTX_BACKEND}.") message(STATUS "Configuring ScriptX option SCRIPTX_NO_EXCEPTION_ON_BIND_FUNCTION ${SCRIPTX_NO_EXCEPTION_ON_BIND_FUNCTION}.") message(STATUS "Configuring ScriptX feature SCRIPTX_FEATURE_INSPECTOR ${SCRIPTX_FEATURE_INSPECTOR}.") -include(${SCRIPTX_DIR}/docs/doxygen/CMakeLists.txt) \ No newline at end of file +include(${SCRIPTX_DIR}/docs/doxygen/CMakeLists.txt) diff --git a/README-zh.md b/README-zh.md index 7bff91ca..ecffd7c4 100644 --- a/README-zh.md +++ b/README-zh.md @@ -125,11 +125,11 @@ ScriptX通过一系列的技术手段实现了脚本的异常和C++异常相互 ScriptX 设计的时候充分考虑到API的易用性，包括操作友好简单，不易出错，错误信息明显，便于定位问题等。在这样的指导思想之下ScriptX做了很多原生引擎做不了的事情。 -比如：V8在destory的时候是不执行GC的，导致很多绑定的native类不能释放。ScriptX做了额外的逻辑处理这个情况。 +比如：V8在destroy的时候是不执行GC的，导致很多绑定的native类不能释放。ScriptX做了额外的逻辑处理这个情况。 V8和JSCore要求在finalize回调中不能调用ScriptX的其他API，否则会crash，这也导致代码逻辑很难实现。ScriptX借助MessageQueue完美规避这个问题。 -V8和JSCore的全局引用都必须在engine destory之前全部释放掉，否则就会引起crash、不能destory等问题。ScriptX则保证在Engine destory的时候主动reset所有 Global / Weak 引用。 +V8和JSCore的全局引用都必须在engine destroy之前全部释放掉，否则就会引起crash、不能destroy等问题。ScriptX则保证在Engine destroy的时候主动reset所有 Global / Weak 引用。 ## 6. 简单高效的绑定API diff --git a/backend/JavaScriptCore/JscEngine.cc b/backend/JavaScriptCore/JscEngine.cc index 9ead88ae..cec728fa 100644 --- a/backend/JavaScriptCore/JscEngine.cc +++ b/backend/JavaScriptCore/JscEngine.cc @@ -19,6 +19,7 @@ #include "../../src/Native.hpp" #include "JscEngine.hpp" #include "JscHelper.h" +#include "../../src/utils/Helper.hpp" namespace script::jsc_backend { @@ -177,6 +178,27 @@ script::Local JscEngine::eval(const script::Local return eval(script, {}); } +Local JscEngine::loadFile(const Local& scriptFile) { + if(scriptFile.toString().empty()) + throw Exception("script file no found"); + Local content = internal::readAllFileContent(scriptFile); + if(content.isNull()) + throw Exception("can't load script file"); + + std::string sourceFilePath = scriptFile.toString(); + std::size_t pathSymbol = sourceFilePath.rfind("/"); + if(pathSymbol != -1) + sourceFilePath = sourceFilePath.substr(pathSymbol + 1); + else + { + pathSymbol = sourceFilePath.rfind("\\"); + if(pathSymbol != -1) + sourceFilePath = sourceFilePath.substr(pathSymbol + 1); + } + Local sourceFileName = String::newString(sourceFilePath); + return eval(content.asString(), sourceFileName); +} + std::shared_ptr JscEngine::messageQueue() { return messageQueue_; } void JscEngine::gc() { diff --git a/backend/JavaScriptCore/JscEngine.h b/backend/JavaScriptCore/JscEngine.h index aedd71ed..0b8f5817 100644 --- a/backend/JavaScriptCore/JscEngine.h +++ b/backend/JavaScriptCore/JscEngine.h @@ -86,6 +86,8 @@ class JscEngine : public ::script::ScriptEngine { Local eval(const Local& script) override; using ScriptEngine::eval; + Local loadFile(const Local& scriptFile) override; + std::shared_ptr messageQueue() override; void gc() override; diff --git a/backend/Lua/LuaEngine.cc b/backend/Lua/LuaEngine.cc index ff86d299..9ce5715e 100644 --- a/backend/Lua/LuaEngine.cc +++ b/backend/Lua/LuaEngine.cc @@ -28,6 +28,7 @@ #include "LuaHelper.hpp" #include "LuaReference.hpp" #include "LuaScope.hpp" +#include "../../src/utils/Helper.hpp" // ref https://www.lua.org/manual/5.1/manual.html // https://www.lua.org/wshop14/Zykov.pdf @@ -259,6 +260,27 @@ Local LuaEngine::eval(const Local& script, const Local& so return lua_backend::callFunction({}, {}, 0, nullptr); } +Local LuaEngine::loadFile(const Local& scriptFile) { + if(scriptFile.toString().empty()) + throw Exception("script file no found"); + Local content = internal::readAllFileContent(scriptFile); + if(content.isNull()) + throw Exception("can't load script file"); + + std::string sourceFilePath = scriptFile.toString(); + std::size_t pathSymbol = sourceFilePath.rfind("/"); + if(pathSymbol != -1) + sourceFilePath = sourceFilePath.substr(pathSymbol + 1); + else + { + pathSymbol = sourceFilePath.rfind("\\"); + if(pathSymbol != -1) + sourceFilePath = sourceFilePath.substr(pathSymbol + 1); + } + Local sourceFileName = String::newString(sourceFilePath); + return eval(content.asString(), sourceFileName); +} + Arguments LuaEngine::makeArguments(LuaEngine* engine, int stackBase, size_t paramCount, bool isInstanceFunc) { lua_backend::ArgumentsData argumentsData{engine, stackBase, paramCount, isInstanceFunc}; diff --git a/backend/Lua/LuaEngine.h b/backend/Lua/LuaEngine.h index 4a1ec3c7..c2cd0135 100644 --- a/backend/Lua/LuaEngine.h +++ b/backend/Lua/LuaEngine.h @@ -77,6 +77,8 @@ class LuaEngine : public ScriptEngine { Local eval(const Local& script) override; using ScriptEngine::eval; + Local loadFile(const Local& scriptFile) override; + std::shared_ptr messageQueue() override; void gc() override; diff --git a/backend/Lua/LuaLocalReference.cc b/backend/Lua/LuaLocalReference.cc index 00788a30..1ab094c7 100644 --- a/backend/Lua/LuaLocalReference.cc +++ b/backend/Lua/LuaLocalReference.cc @@ -43,6 +43,28 @@ void ensureNonnull(int index) { throw Exception("NullPointerException"); } +bool judgeIsArray(int index) +{ + auto lua = currentLua(); + int currectArrIndex = 0; + + lua_pushnil(lua); + + while (lua_next(lua, index)) + { + // Copy current key and judge it's type + lua_pushvalue(lua, -2); + if(!lua_isnumber(lua,-1) || lua_tonumber(lua,-1) != ++currectArrIndex) + { + lua_pop(lua, 3); + return false; + } + lua_pop(lua, 2); + } + return true; +} + + } // namespace lua_backend #define REF_IMPL_BASIC_FUNC(ValueType) \ @@ -149,8 +171,11 @@ ValueKind Local::getKind() const { } else if (isByteBuffer()) { return ValueKind::kByteBuffer; } else if (type == LUA_TTABLE) { - // lua don't have array type, the are all tables - return ValueKind::kObject; + // Traverse the table to judge whether it is an array + if(isArray()) + return ValueKind::kArray; + else + return ValueKind::kObject; } else { return ValueKind::kUnsupported; } @@ -176,7 +201,11 @@ bool Local::isFunction() const { return val_ != 0 && lua_type(lua_backend::currentLua(), val_) == LUA_TFUNCTION; } -bool Local::isArray() const { return isObject(); } +bool Local::isArray() const { + if(val_ == 0 || lua_type(lua_backend::currentLua(), val_) != LUA_TTABLE) + return false; + return lua_backend::judgeIsArray(val_); +} bool Local::isByteBuffer() const { auto engine = lua_backend::currentEngine(); @@ -437,4 +466,4 @@ void Local::commit() const {} void Local::sync() const {} -} // namespace script +} // namespace script \ No newline at end of file diff --git a/backend/Python/PyEngine.cc b/backend/Python/PyEngine.cc index 2d3c2dea..521c8a34 100644 --- a/backend/Python/PyEngine.cc +++ b/backend/Python/PyEngine.cc @@ -16,24 +16,87 @@ */ #include "PyEngine.h" +#include #include "../../src/Utils.h" +#include "../../src/utils/Helper.hpp" namespace script::py_backend { PyEngine::PyEngine(std::shared_ptr queue) : queue_(queue ? std::move(queue) : std::make_shared()) { - Py_Initialize(); + if (Py_IsInitialized() == 0) { + Py_SetStandardStreamEncoding("utf-8", nullptr); + // Python not initialized. Init main interpreter + Py_InitializeEx(0); + // Init threading environment + PyEval_InitThreads(); + // Initialize type + namespaceType_ = makeNamespaceType(); + staticPropertyType_ = makeStaticPropertyType(); + defaultMetaType_ = makeDefaultMetaclass(); + // Save main thread state & release GIL + mainThreadState_ = PyEval_SaveThread(); + } + + // Resume main thread state (to execute Py_NewInterpreter) + PyThreadState* oldState = PyThreadState_Swap(mainThreadState_); + + // If GIL is released, lock it + if (PyEngine::engineEnterCount_ == 0) { + PyEval_AcquireLock(); + } + // Create new interpreter + PyThreadState* newSubState = Py_NewInterpreter(); + if (!newSubState) { + throw Exception("Fail to create sub interpreter"); + } + subInterpreterState_ = newSubState->interp; + + // If GIL is released before, unlock it + if (PyEngine::engineEnterCount_ == 0) { + PyEval_ReleaseLock(); + } + // Store created new sub thread state & recover old thread state stored before + subThreadState_.set(PyThreadState_Swap(oldState)); } PyEngine::PyEngine() : PyEngine(nullptr) {} PyEngine::~PyEngine() = default; -void PyEngine::destroy() noexcept { ScriptEngine::destroyUserData(); } +void PyEngine::destroy() noexcept { + ScriptEngine::destroyUserData(); // TODO: solve this problem about Py_EndInterpreter + /*if (PyEngine::engineEnterCount_ == 0) { + // GIL is not locked. Just lock it + PyEval_AcquireLock(); + } + // Swap to clear thread state & end sub interpreter + PyThreadState* oldThreadState = PyThreadState_Swap(subThreadState_.get()); + Py_EndInterpreter(subThreadState_.get()); + // Recover old thread state + PyThreadState_Swap(oldThreadState); + + if (PyEngine::engineEnterCount_ == 0) { + // Unlock the GIL because it is not locked before + PyEval_ReleaseLock(); + }*/ +} -Local PyEngine::get(const Local& key) { return Local(); } +Local PyEngine::get(const Local& key) { + PyObject* item = PyDict_GetItemString(getGlobalDict(), key.toStringHolder().c_str()); + if (item) + return py_interop::toLocal(item); + else + return py_interop::toLocal(Py_None); +} -void PyEngine::set(const Local& key, const Local& value) {} +void PyEngine::set(const Local& key, const Local& value) { + int result = + PyDict_SetItemString(getGlobalDict(), key.toStringHolder().c_str(), py_interop::getPy(value)); + if (result != 0) { + checkPyErr(); + } +} Local PyEngine::eval(const Local& script) { return eval(script, Local()); } @@ -42,7 +105,38 @@ Local PyEngine::eval(const Local& script, const Local& so } Local PyEngine::eval(const Local& script, const Local& sourceFile) { - return Local(); + // Limitation: one line code must be expression (no "\n", no "=") + const char* source = script.toStringHolder().c_str(); + bool oneLine = true; + if (strstr(source, "\n") != NULL) + oneLine = false; + else if (strstr(source, " = ") != NULL) + oneLine = false; + PyObject* result = PyRun_StringFlags(source, oneLine ? Py_eval_input : Py_file_input, + getGlobalDict(), nullptr, nullptr); + checkPyErr(); + return py_interop::asLocal(result); +} + +Local PyEngine::loadFile(const Local& scriptFile) { + std::string sourceFilePath = scriptFile.toString(); + if (sourceFilePath.empty()) { + throw Exception("script file no found"); + } + Local content = internal::readAllFileContent(scriptFile); + if (content.isNull()) { + throw Exception("can't load script file"); + } + + std::size_t pathSymbol = sourceFilePath.rfind("/"); + if (pathSymbol != -1) { + sourceFilePath = sourceFilePath.substr(pathSymbol + 1); + } else { + pathSymbol = sourceFilePath.rfind("\\"); + if (pathSymbol != -1) sourceFilePath = sourceFilePath.substr(pathSymbol + 1); + } + Local sourceFileName = String::newString(sourceFilePath); + return eval(content.asString(), sourceFileName); } std::shared_ptr PyEngine::messageQueue() { return queue_; } @@ -56,5 +150,4 @@ ScriptLanguage PyEngine::getLanguageType() { return ScriptLanguage::kPython; } std::string PyEngine::getEngineVersion() { return Py_GetVersion(); } bool PyEngine::isDestroying() const { return false; } - } // namespace script::py_backend diff --git a/backend/Python/PyEngine.h b/backend/Python/PyEngine.h index e1c57249..b1255642 100644 --- a/backend/Python/PyEngine.h +++ b/backend/Python/PyEngine.h @@ -17,18 +17,46 @@ #pragma once +#include #include "../../src/Engine.h" #include "../../src/Exception.h" +#include "../../src/utils/Helper.hpp" #include "../../src/utils/MessageQueue.h" -#include "PyHelper.h" +#include "PyHelper.hpp" namespace script::py_backend { +// an PyEngine = a subinterpreter class PyEngine : public ScriptEngine { private: std::shared_ptr<::script::utils::MessageQueue> queue_; + std::unordered_map registeredTypes_; + std::unordered_map registeredTypesReverse_; + + // Global thread state of main interpreter + inline static PyThreadState* mainThreadState_ = nullptr; + // Sub interpreter storage + PyInterpreterState* subInterpreterState_; + // Sub thread state of this sub interpreter (in TLS) + TssStorage subThreadState_; + + // When you use EngineScope to enter a new engine(subinterpreter) + // and find that there is an existing thread state owned by another engine, + // we need to push its thread state to stack and release GIL to avoid dead-lock + // -- see more code in "PyScope.cc" + std::stack oldThreadStateStack_; + + // Record global EngineScope enter times to determine + // whether it is needed to unlock GIL when exit EngineScope + // -- see more comments in "PyScope.cc" + inline static int engineEnterCount_ = 0; + public: + inline static PyTypeObject* staticPropertyType_ = nullptr; + inline static PyTypeObject* namespaceType_ = nullptr; + inline static PyTypeObject* defaultMetaType_ = nullptr; + PyEngine(std::shared_ptr<::script::utils::MessageQueue> queue); PyEngine(); @@ -49,6 +77,8 @@ class PyEngine : public ScriptEngine { Local eval(const Local& script) override; using ScriptEngine::eval; + Local loadFile(const Local& scriptFile) override; + std::shared_ptr messageQueue() override; void gc() override; @@ -64,28 +94,393 @@ class PyEngine : public ScriptEngine { private: template - bool registerNativeClassImpl(const ClassDefine* classDefine) { - return false; + void nameSpaceSet(const ClassDefine* classDefine, const std::string& name, PyObject* value) { + std::string nameSpace = classDefine->getNameSpace(); + PyObject* nameSpaceObj = getGlobalDict(); + + if (nameSpace.empty()) { + PyDict_SetItemString(nameSpaceObj, name.c_str(), value); + } else { // namespace can be aaa.bbb.ccc + std::size_t begin = 0; + while (begin < nameSpace.size()) { + auto index = nameSpace.find('.', begin); + if (index == std::string::npos) { + index = nameSpace.size(); + } + + PyObject* sub = nullptr; + auto key = nameSpace.substr(begin, index - begin); + if (PyDict_CheckExact(nameSpaceObj)) { + sub = PyDict_GetItemString(nameSpaceObj, key.c_str()); + if (sub == nullptr) { + PyObject* args = PyTuple_New(0); + PyTypeObject* type = reinterpret_cast(namespaceType_); + sub = type->tp_new(type, args, nullptr); + Py_DECREF(args); + PyDict_SetItemString(nameSpaceObj, key.c_str(), sub); + } + setAttr(sub, name.c_str(), value); + } else /*namespace type*/ { + if (hasAttr(nameSpaceObj, key.c_str())) { + sub = getAttr(nameSpaceObj, key.c_str()); + } else { + PyObject* args = PyTuple_New(0); + PyTypeObject* type = reinterpret_cast(namespaceType_); + sub = type->tp_new(type, args, nullptr); + Py_DECREF(args); + setAttr(nameSpaceObj, key.c_str(), sub); + } + setAttr(sub, name.c_str(), value); + } + nameSpaceObj = sub; + begin = index + 1; + } + } + } + + inline PyObject* warpGetter(const char* name, GetterCallback callback) { + struct FunctionData { + GetterCallback function; + PyEngine* engine; + }; + + PyMethodDef* method = new PyMethodDef; + method->ml_name = name; + method->ml_flags = METH_VARARGS; + method->ml_doc = nullptr; + method->ml_meth = [](PyObject* self, PyObject* args) -> PyObject* { + auto data = static_cast(PyCapsule_GetPointer(self, nullptr)); + try { + return py_interop::peekPy(data->function()); + } catch (const Exception& e) { + rethrowException(e); + } + return nullptr; + }; + + PyCapsule_Destructor destructor = [](PyObject* cap) { + void* ptr = PyCapsule_GetPointer(cap, nullptr); + delete static_cast(ptr); + }; + PyObject* capsule = + PyCapsule_New(new FunctionData{std::move(callback), this}, nullptr, destructor); + checkPyErr(); + + PyObject* function = PyCFunction_New(method, capsule); + Py_DECREF(capsule); + checkPyErr(); + return function; + } + + template + inline PyObject* warpInstanceGetter(const char* name, InstanceGetterCallback callback) { + struct FunctionData { + InstanceGetterCallback function; + PyEngine* engine; + }; + + PyMethodDef* method = new PyMethodDef; + method->ml_name = name; + method->ml_flags = METH_VARARGS; + method->ml_doc = nullptr; + method->ml_meth = [](PyObject* self, PyObject* args) -> PyObject* { + auto data = static_cast(PyCapsule_GetPointer(self, nullptr)); + try { + T* thiz = GeneralObject::getInstance(PyTuple_GetItem(args, 0)); + return py_interop::peekPy(data->function(thiz)); + } catch (const Exception& e) { + rethrowException(e); + } + return nullptr; + }; + + PyCapsule_Destructor destructor = [](PyObject* cap) { + void* ptr = PyCapsule_GetPointer(cap, nullptr); + delete static_cast(ptr); + }; + PyObject* capsule = + PyCapsule_New(new FunctionData{std::move(callback), this}, nullptr, destructor); + checkPyErr(); + + PyObject* function = PyCFunction_New(method, capsule); + Py_DECREF(capsule); + checkPyErr(); + + return function; + } + + inline PyObject* warpSetter(const char* name, SetterCallback callback) { + struct FunctionData { + SetterCallback function; + PyEngine* engine; + }; + + PyMethodDef* method = new PyMethodDef; + method->ml_name = name; + method->ml_flags = METH_VARARGS; + method->ml_doc = nullptr; + method->ml_meth = [](PyObject* self, PyObject* args) -> PyObject* { + auto data = static_cast(PyCapsule_GetPointer(self, nullptr)); + try { + data->function(py_interop::toLocal(PyTuple_GetItem(args, 1))); + Py_RETURN_NONE; + } catch (const Exception& e) { + rethrowException(e); + } + return nullptr; + }; + + PyCapsule_Destructor destructor = [](PyObject* cap) { + void* ptr = PyCapsule_GetPointer(cap, nullptr); + delete static_cast(ptr); + }; + PyObject* capsule = + PyCapsule_New(new FunctionData{std::move(callback), this}, nullptr, destructor); + checkPyErr(); + + PyObject* function = PyCFunction_New(method, capsule); + Py_DECREF(capsule); + checkPyErr(); + + return function; + } + + template + inline PyObject* warpInstanceSetter(const char* name, InstanceSetterCallback callback) { + struct FunctionData { + InstanceSetterCallback function; + PyEngine* engine; + }; + + PyMethodDef* method = new PyMethodDef; + method->ml_name = name; + method->ml_flags = METH_VARARGS; + method->ml_doc = nullptr; + method->ml_meth = [](PyObject* self, PyObject* args) -> PyObject* { + auto data = static_cast(PyCapsule_GetPointer(self, nullptr)); + try { + T* thiz = GeneralObject::getInstance(PyTuple_GetItem(args, 0)); + data->function(thiz, py_interop::toLocal(PyTuple_GetItem(args, 1))); + Py_RETURN_NONE; + } catch (const Exception& e) { + rethrowException(e); + } + return nullptr; + }; + + PyCapsule_Destructor destructor = [](PyObject* cap) { + void* ptr = PyCapsule_GetPointer(cap, nullptr); + delete static_cast(ptr); + }; + PyObject* capsule = + PyCapsule_New(new FunctionData{std::move(callback), this}, nullptr, destructor); + checkPyErr(); + + PyObject* function = PyCFunction_New(method, capsule); + Py_DECREF(capsule); + checkPyErr(); + + return function; } - Local getNamespaceForRegister(const std::string_view& nameSpace) { - TEMPLATE_NOT_IMPLEMENTED(); + template + inline void registerStaticProperty(const ClassDefine* classDefine, PyObject* type) { + for (const auto& property : classDefine->staticDefine.properties) { + PyObject* doc = toStr(""); + PyObject* args = + PyTuple_Pack(4, warpGetter(property.name.c_str(), property.getter), + warpSetter(property.name.c_str(), property.setter), Py_None, doc); + Py_DECREF(doc); + PyObject* warpped_property = PyObject_Call((PyObject*)staticPropertyType_, args, nullptr); + setAttr(type, property.name.c_str(), warpped_property); + } + } + + template + inline void registerInstanceProperty(const ClassDefine* classDefine, PyObject* type) { + for (const auto& property : classDefine->instanceDefine.properties) { + PyObject* doc = toStr(""); + PyObject* args = + PyTuple_Pack(4, warpInstanceGetter(property.name.c_str(), property.getter), + warpInstanceSetter(property.name.c_str(), property.setter), Py_None, doc); + Py_DECREF(doc); + PyObject* warpped_property = PyObject_Call((PyObject*)&PyProperty_Type, args, nullptr); + setAttr(type, property.name.c_str(), warpped_property); + } + } + + template + inline void registerStaticFunction(const ClassDefine* classDefine, PyObject* type) { + for (const auto& f : classDefine->staticDefine.functions) { + struct FunctionData { + FunctionCallback function; + py_backend::PyEngine* engine; + }; + + PyMethodDef* method = new PyMethodDef; + method->ml_name = f.name.c_str(); + method->ml_flags = METH_VARARGS; + method->ml_doc = nullptr; + method->ml_meth = [](PyObject* self, PyObject* args) -> PyObject* { + auto data = static_cast(PyCapsule_GetPointer(self, nullptr)); + try { + return py_interop::peekPy( + data->function(py_interop::makeArguments(data->engine, self, args))); + } catch (const Exception& e) { + rethrowException(e); + } + return nullptr; + }; + + PyCapsule_Destructor destructor = [](PyObject* cap) { + void* ptr = PyCapsule_GetPointer(cap, nullptr); + delete static_cast(ptr); + }; + PyObject* capsule = + PyCapsule_New(new FunctionData{std::move(f.callback), this}, nullptr, destructor); + checkPyErr(); + + PyObject* function = PyCFunction_New(method, capsule); + Py_DECREF(capsule); + checkPyErr(); + + setAttr(type, f.name.c_str(), PyStaticMethod_New(function)); + } + } + + template + inline void registerInstanceFunction(const ClassDefine* classDefine, PyObject* type) { + for (const auto& f : classDefine->instanceDefine.functions) { + struct FunctionData { + InstanceFunctionCallback function; + py_backend::PyEngine* engine; + }; + + PyMethodDef* method = new PyMethodDef; + method->ml_name = f.name.c_str(); + method->ml_flags = METH_VARARGS; + method->ml_doc = nullptr; + method->ml_meth = [](PyObject* self, PyObject* args) -> PyObject* { + auto data = static_cast(PyCapsule_GetPointer(self, nullptr)); + try { + T* thiz = GeneralObject::getInstance(PyTuple_GetItem(args, 0)); + PyObject* real_args = PyTuple_GetSlice(args, 1, PyTuple_Size(args)); + auto ret = data->function(thiz, py_interop::makeArguments(data->engine, self, real_args)); + Py_DECREF(real_args); + return py_interop::peekPy(ret); + } catch (const Exception& e) { + rethrowException(e); + } + return nullptr; + }; + + PyCapsule_Destructor destructor = [](PyObject* cap) { + void* ptr = PyCapsule_GetPointer(cap, nullptr); + delete static_cast(ptr); + }; + PyObject* capsule = + PyCapsule_New(new FunctionData{std::move(f.callback), this}, nullptr, destructor); + checkPyErr(); + + PyObject* function = PyCFunction_New(method, capsule); + Py_DECREF(capsule); + checkPyErr(); + setAttr(type, f.name.c_str(), PyInstanceMethod_New(function)); + } + } + + template + void registerNativeClassImpl(const ClassDefine* classDefine) { + bool constructable = bool(classDefine->instanceDefine.constructor); + + auto name_obj = toStr(classDefine->className.c_str()); + + auto* heap_type = (PyHeapTypeObject*)PyType_GenericAlloc(PyEngine::defaultMetaType_, 0); + if (!heap_type) { + Py_FatalError("error allocating type!"); + } + + heap_type->ht_name = Py_NewRef(name_obj); + heap_type->ht_qualname = Py_NewRef(name_obj); + + auto* type = &heap_type->ht_type; + type->tp_name = classDefine->className.c_str(); + Py_INCREF(&PyBaseObject_Type); + type->tp_base = &PyBaseObject_Type; + type->tp_basicsize = static_cast(sizeof(GeneralObject)); + type->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HEAPTYPE; + + type->tp_new = [](PyTypeObject* type, PyObject* args, PyObject* kwds) -> PyObject* { + PyObject* self = type->tp_alloc(type, 0); + auto* thiz = reinterpret_cast(self); + auto engine = currentEngine(); + auto classDefine = + reinterpret_cast*>(engine->registeredTypesReverse_[self->ob_type]); + if (classDefine->instanceDefine.constructor) { + thiz->instance = + classDefine->instanceDefine.constructor(py_interop::makeArguments(engine, self, args)); + } else { + throw Exception("the class has no constructor"); + } + return self; + }; + type->tp_init = [](PyObject* self, PyObject*, PyObject*) -> int { + PyTypeObject* type = Py_TYPE(self); + std::string msg = std::string(type->tp_name) + ": No constructor defined!"; + PyErr_SetString(PyExc_TypeError, msg.c_str()); + return -1; + }; + type->tp_dealloc = [](PyObject* self) { + auto* type = Py_TYPE(self); + type->tp_free(self); + Py_DECREF(type); + }; + + /* Support weak references (needed for the keep_alive feature) */ + type->tp_weaklistoffset = offsetof(GeneralObject, weakrefs); + + if (PyType_Ready(type) < 0) { + Py_FatalError("PyType_Ready failed in make_object_base_type()"); + } + + setAttr((PyObject*)type, "__module__", toStr("scriptx_builtins")); + + this->registerStaticProperty(classDefine, (PyObject*)type); + this->registerStaticFunction(classDefine, (PyObject*)type); + if (constructable) { + this->registerInstanceProperty(classDefine, (PyObject*)type); + this->registerInstanceFunction(classDefine, (PyObject*)type); + } + this->registeredTypes_.emplace(classDefine, type); + this->registeredTypesReverse_.emplace(type, classDefine); + this->nameSpaceSet(classDefine, classDefine->className.c_str(), (PyObject*)type); } template Local newNativeClassImpl(const ClassDefine* classDefine, size_t size, const Local* args) { - TEMPLATE_NOT_IMPLEMENTED(); + PyObject* tuple = PyTuple_New(size); + for (size_t i = 0; i < size; ++i) { + PyTuple_SetItem(tuple, i, py_interop::getPy(args[i])); + } + + PyTypeObject* type = registeredTypes_[classDefine]; + PyObject* obj = type->tp_new(type, tuple, nullptr); + Py_DECREF(tuple); + return Local(obj); } template bool isInstanceOfImpl(const Local& value, const ClassDefine* classDefine) { - return false; + return registeredTypes_[classDefine] == py_interop::peekPy(value)->ob_type; } template T* getNativeInstanceImpl(const Local& value, const ClassDefine* classDefine) { - return nullptr; + if (!isInstanceOfImpl(value, classDefine)) { + throw Exception("Unmatched type of the value!"); + } + return GeneralObject::getInstance(py_interop::peekPy(value)); } private: @@ -113,6 +508,12 @@ class PyEngine : public ScriptEngine { friend class ::script::Arguments; friend class ::script::ScriptClass; + + friend class EngineScopeImpl; + + friend class ExitEngineScopeImpl; + + friend PyTypeObject* makeDefaultMetaclass(); }; } // namespace script::py_backend \ No newline at end of file diff --git a/backend/Python/PyException.cc b/backend/Python/PyException.cc index 871a0f3c..323fa6cd 100644 --- a/backend/Python/PyException.cc +++ b/backend/Python/PyException.cc @@ -16,25 +16,109 @@ */ #include +#include "PyHelper.h" namespace script { -Exception::Exception(std::string msg) : std::exception(), exception_({}) { - exception_.message_ = std::move(msg); +namespace py_backend { + +void ExceptionFields::fillMessage() const noexcept { + if (exception_.isEmpty() || exception_.getValue().isString()) { + return; + } + PyObject *capsule = py_interop::peekPy(exception_.getValue()); + if (!PyCapsule_IsValid(capsule, nullptr)) { + return; + } + ExceptionInfo *errStruct = + (ExceptionInfo *)PyCapsule_GetPointer(capsule, nullptr); + + PyTypeObject *typeObj = (PyTypeObject *)(errStruct->pType); + PyObject *formattedMsg = PyObject_Str(errStruct->pValue); + if (!formattedMsg) { + return; + } + // NameError: name 'hello' is not defined + message_ = std::string(typeObj->tp_name) + ": " + PyUnicode_AsUTF8(formattedMsg); + hasMessage_ = true; } -Exception::Exception(const script::Local& message) - : std::exception(), exception_() {} +void ExceptionFields::fillStacktrace() const noexcept { + if (exception_.isEmpty() || exception_.getValue().isString()) { + return; + } + PyObject *capsule = py_interop::peekPy(exception_.getValue()); + if (!PyCapsule_IsValid(capsule, nullptr)) { + return; + } + ExceptionInfo *errStruct = + (ExceptionInfo *)PyCapsule_GetPointer(capsule, nullptr); + + PyTracebackObject *tb = (PyTracebackObject *)(errStruct->pTraceback); + if (tb == nullptr) + return; + + // Get the deepest trace possible. + while (tb->tb_next) { + tb = tb->tb_next; + } + PyFrameObject *frame = tb->tb_frame; + Py_XINCREF(frame); + stacktrace_ = "Traceback (most recent call last):"; + while (frame) { + stacktrace_ += '\n'; + PyCodeObject *f_code = PyFrame_GetCode(frame); + int lineno = PyFrame_GetLineNumber(frame); + stacktrace_ += " File \""; + stacktrace_ += PyUnicode_AsUTF8(f_code->co_filename); + stacktrace_ += "\", line "; + stacktrace_ += std::to_string(lineno); + stacktrace_ += ", in "; + stacktrace_ += PyUnicode_AsUTF8(f_code->co_name); + Py_DECREF(f_code); + frame = frame->f_back; + } + hasStacktrace_ = true; +} -Exception::Exception(const script::Local& exception) - : std::exception(), exception_({}) {} +} // namespace py_backend -Local Exception::exception() const { return {}; } +Exception::Exception(std::string msg) : std::exception(), exception_() { + exception_.message_ = msg; + exception_.hasMessage_ = true; +} -std::string Exception::message() const noexcept { return exception_.message_; } +Exception::Exception(const script::Local &message) + : std::exception(), exception_() { + exception_.exception_ = message; + exception_.hasMessage_ = true; +} -std::string Exception::stacktrace() const noexcept { return "[no stacktrace]"; } +Exception::Exception(const script::Local &exception) + : std::exception(), exception_({}) { + exception_.exception_ = exception; +} -const char* Exception::what() const noexcept { return exception_.message_.c_str(); } +Local Exception::exception() const { + if (exception_.exception_.isEmpty()) { + exception_.exception_ = String::newString(exception_.message_); + } + return exception_.exception_.getValue(); +} + +std::string Exception::message() const noexcept { + exception_.fillMessage(); + return exception_.hasMessage_ ? exception_.message_ : "[No Exception Message]"; +} + +std::string Exception::stacktrace() const noexcept { + exception_.fillStacktrace(); + return exception_.hasStacktrace_ ? exception_.stacktrace_ : "[No Stacktrace]"; +} + +const char *Exception::what() const noexcept { + exception_.fillMessage(); + return exception_.hasMessage_ ? exception_.message_.c_str() : "[No Exception Message]"; +} } // namespace script diff --git a/backend/Python/PyHelper.cc b/backend/Python/PyHelper.cc index ced86204..5f77c852 100644 --- a/backend/Python/PyHelper.cc +++ b/backend/Python/PyHelper.cc @@ -16,25 +16,272 @@ */ #include "PyHelper.hpp" +#include "PyEngine.h" namespace script::py_backend { -PyObject* checkException(PyObject* obj) { - if (!obj) { - checkException(); +void setAttr(PyObject* obj, PyObject* key, PyObject* value) { + if (PyObject_SetAttr(obj, key, value) != 0) { + throw Exception(); } - return obj; } -void checkException() { - auto err = PyErr_Occurred(); - if (err) { - // TODO +void setAttr(PyObject* obj, const char* key, PyObject* value) { + if (PyObject_SetAttrString(obj, key, value) != 0) { + throw Exception(); } } -void rethrowException(const Exception& exception) { - // TODO +PyObject* getAttr(PyObject* obj, PyObject* key) { + PyObject* result = PyObject_GetAttr(obj, key); + if (!result) { + throw Exception(); + } + return result; +} + +PyObject* getAttr(PyObject* obj, const char* key) { + PyObject* result = PyObject_GetAttrString(obj, key); + if (!result) { + throw Exception(); + } + return result; +} + +bool hasAttr(PyObject* obj, PyObject* key) { return PyObject_HasAttr(obj, key) == 1; } + +bool hasAttr(PyObject* obj, const char* key) { return PyObject_HasAttrString(obj, key) == 1; } + +void delAttr(PyObject* obj, PyObject* key) { + if (PyObject_DelAttr(obj, key) != 0) { + throw Exception(); + } +} + +void delAttr(PyObject* obj, const char* key) { + if (PyObject_DelAttrString(obj, key) != 0) { + throw Exception(); + } +} + +PyObject* toStr(const char* s) { return PyUnicode_FromString(s); } + +PyObject* toStr(const std::string& s) { return PyUnicode_FromStringAndSize(s.c_str(), s.size()); } + +void checkPyErr() { + if (PyErr_Occurred()) { + PyObject *pType, *pValue, *pTraceback; + PyErr_Fetch(&pType, &pValue, &pTraceback); + PyErr_NormalizeException(&pType, &pValue, &pTraceback); + + ExceptionInfo* errStruct = new ExceptionInfo; + errStruct->pType = pType; + errStruct->pValue = pValue; + errStruct->pTraceback = pTraceback; + + PyObject* capsule = PyCapsule_New(errStruct, nullptr, [](PyObject* cap) { + void* ptr = PyCapsule_GetPointer(cap, nullptr); + delete static_cast(ptr); + }); + + if (!capsule) return; + throw Exception(py_interop::asLocal(capsule)); + } +} + +void rethrowException(const Exception& exception) { throw exception; } + +PyEngine* currentEngine() { return EngineScope::currentEngineAs(); } + +PyEngine* currentEngineChecked() { return &EngineScope::currentEngineCheckedAs(); } + +PyObject* getGlobalDict() { + PyObject* m = PyImport_AddModule("__main__"); + if (m == nullptr) { + throw Exception("can't find __main__ module"); + } + return PyModule_GetDict(m); +} + +inline PyObject* scriptx_static_get(PyObject* self, PyObject* /*ob*/, PyObject* cls) { + return PyProperty_Type.tp_descr_get(self, cls, cls); +} + +inline int scriptx_static_set(PyObject* self, PyObject* obj, PyObject* value) { + PyObject* cls = PyType_Check(obj) ? obj : (PyObject*)Py_TYPE(obj); + return PyProperty_Type.tp_descr_set(self, cls, value); +} + +inline PyObject* scriptx_get_dict(PyObject* self, void*) { + PyObject*& dict = *_PyObject_GetDictPtr(self); + if (!dict) { + dict = PyDict_New(); + } + Py_XINCREF(dict); + return dict; +} + +inline int scriptx_set_dict(PyObject* self, PyObject* new_dict, void*) { + if (!PyDict_Check(new_dict)) { + PyErr_SetString(PyExc_TypeError, "__dict__ must be set to a dictionary"); + return -1; + } + PyObject*& dict = *_PyObject_GetDictPtr(self); + Py_INCREF(new_dict); + Py_CLEAR(dict); + dict = new_dict; + return 0; +} + +inline int scriptx_traverse(PyObject* self, visitproc visit, void* arg) { + PyObject*& dict = *_PyObject_GetDictPtr(self); + Py_VISIT(dict); +// https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_traverse +#if PY_VERSION_HEX >= 0x03090000 + Py_VISIT(Py_TYPE(self)); +#endif + return 0; +} + +inline int scriptx_clear(PyObject* self) { + PyObject*& dict = *_PyObject_GetDictPtr(self); + Py_CLEAR(dict); + return 0; +} + +PyTypeObject* makeStaticPropertyType() { + constexpr auto* name = "static_property"; + + auto* heap_type = (PyHeapTypeObject*)PyType_Type.tp_alloc(&PyType_Type, 0); + if (!heap_type) { + Py_FatalError("error allocating type!"); + } + + heap_type->ht_name = PyUnicode_InternFromString(name); + heap_type->ht_qualname = PyUnicode_InternFromString(name); + + auto* type = &heap_type->ht_type; + type->tp_name = name; + type->tp_base = &PyProperty_Type; + type->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HEAPTYPE; + type->tp_descr_get = &scriptx_static_get; + type->tp_descr_set = &scriptx_static_set; + + if (PyType_Ready(type) < 0) { + Py_FatalError("failure in PyType_Ready()!"); + } + + setAttr((PyObject*)type, "__module__", PyUnicode_InternFromString("scriptx_builtins")); + + return type; +} + +PyTypeObject* makeNamespaceType() { + constexpr auto* name = "namespace"; + + auto* heap_type = (PyHeapTypeObject*)PyType_Type.tp_alloc(&PyType_Type, 0); + if (!heap_type) { + Py_FatalError("error allocating type!"); + } + + heap_type->ht_name = PyUnicode_InternFromString(name); + heap_type->ht_qualname = PyUnicode_InternFromString(name); + + auto* type = &heap_type->ht_type; + type->tp_name = name; + type->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HEAPTYPE; + + type->tp_dictoffset = PyBaseObject_Type.tp_basicsize; // place dict at the end + type->tp_basicsize = + PyBaseObject_Type.tp_basicsize + sizeof(PyObject*); // and allocate enough space for it + type->tp_traverse = scriptx_traverse; + type->tp_clear = scriptx_clear; + + static PyGetSetDef getset[] = {{"__dict__", scriptx_get_dict, scriptx_set_dict, nullptr, nullptr}, + {nullptr, nullptr, nullptr, nullptr, nullptr}}; + type->tp_getset = getset; + + if (PyType_Ready(type) < 0) { + Py_FatalError("failure in PyType_Ready()!"); + } + setAttr((PyObject*)type, "__module__", PyUnicode_InternFromString("scriptx_builtins")); + + return type; +} + +PyTypeObject* makeDefaultMetaclass() { + constexpr auto* name = "scriptx_type"; + auto name_obj = toStr(name); + + auto* heap_type = (PyHeapTypeObject*)PyType_Type.tp_alloc(&PyType_Type, 0); + if (!heap_type) { + Py_FatalError("error allocating type!"); + } + + heap_type->ht_name = Py_NewRef(name_obj); + heap_type->ht_qualname = Py_NewRef(name_obj); + + auto* type = &heap_type->ht_type; + type->tp_name = name; + Py_INCREF(&PyType_Type); + type->tp_base = &PyType_Type; + type->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HEAPTYPE; + + type->tp_call = [](PyObject* type, PyObject* args, PyObject* kwargs) -> PyObject* { + // use the default metaclass call to create/initialize the object + PyObject* self = PyType_Type.tp_call(type, args, kwargs); + if (self == nullptr) { + return nullptr; + } + return self; + }; + + type->tp_setattro = [](PyObject* obj, PyObject* name, PyObject* value) { + // Use `_PyType_Lookup()` instead of `PyObject_GetAttr()` in order to get the raw + // descriptor (`property`) instead of calling `tp_descr_get` (`property.__get__()`). + PyObject* descr = _PyType_Lookup((PyTypeObject*)obj, name); + + // The following assignment combinations are possible: + // 1. `Type.static_prop = value` --> descr_set: `Type.static_prop.__set__(value)` + // 2. `Type.static_prop = other_static_prop` --> setattro: replace existing `static_prop` + // 3. `Type.regular_attribute = value` --> setattro: regular attribute assignment + auto* const static_prop = (PyObject*)PyEngine::staticPropertyType_; + const auto call_descr_set = (descr != nullptr) && (value != nullptr) && + (PyObject_IsInstance(descr, static_prop) != 0) && + (PyObject_IsInstance(value, static_prop) == 0); + if (call_descr_set) { + // Call `static_property.__set__()` instead of replacing the `static_property`. + return Py_TYPE(descr)->tp_descr_set(descr, obj, value); + } else { + // Replace existing attribute. + return PyType_Type.tp_setattro(obj, name, value); + } + }; + type->tp_getattro = [](PyObject* obj, PyObject* name) { + PyObject* descr = _PyType_Lookup((PyTypeObject*)obj, name); + if (descr && PyInstanceMethod_Check(descr)) { + Py_INCREF(descr); + return descr; + } + return PyType_Type.tp_getattro(obj, name); + }; + + type->tp_dealloc = [](PyObject* obj) { + auto* type = (PyTypeObject*)obj; + auto engine = currentEngine(); + + engine->registeredTypes_.erase(type); + engine->registeredTypesReverse_.erase(type); + PyType_Type.tp_dealloc(obj); + }; + + if (PyType_Ready(type) < 0) { + Py_FatalError("make_default_metaclass(): failure in PyType_Ready()!"); + } + + setAttr((PyObject*)type, "__module__", toStr("scriptx_builtins")); + + return type; } -} // namespace script::py_backend \ No newline at end of file +} // namespace script::py_backend diff --git a/backend/Python/PyHelper.h b/backend/Python/PyHelper.h index fb9b5d81..4c1b0918 100644 --- a/backend/Python/PyHelper.h +++ b/backend/Python/PyHelper.h @@ -25,19 +25,53 @@ // https://docs.python.org/3.8/c-api/init.html#thread-state-and-the-global-interpreter-lock SCRIPTX_BEGIN_INCLUDE_LIBRARY -#ifndef PY_SSIZE_T_CLEAN -#define PY_SSIZE_T_CLEAN -#endif #include -#include +#include +#include SCRIPTX_END_INCLUDE_LIBRARY +#if PY_VERSION_HEX < 0x030a00f0 +#error "python version must be greater than 3.10.0" +#endif + namespace script::py_backend { +struct ExceptionInfo { + PyObject* pType; + PyObject* pValue; + PyObject* pTraceback; +}; + +struct GeneralObject : PyObject { + void* instance; + PyObject* weakrefs; + + template + static T* getInstance(PyObject* self) { + return reinterpret_cast(reinterpret_cast(self)->instance); + } + +}; + +void setAttr(PyObject* obj, PyObject* key, PyObject* value); +void setAttr(PyObject* obj, const char* key, PyObject* value); +PyObject* getAttr(PyObject* obj, PyObject* key); +PyObject* getAttr(PyObject* obj, const char* key); +bool hasAttr(PyObject* obj, PyObject* key); +bool hasAttr(PyObject* obj, const char* key); +void delAttr(PyObject* obj, PyObject* key); +void delAttr(PyObject* obj, const char* key); + +PyObject* toStr(const char* s); +PyObject* toStr(const std::string& s); + class PyEngine; -PyObject* checkException(PyObject* obj); -void checkException(); +void checkPyErr(); void rethrowException(const Exception& exception); +PyEngine* currentEngine(); +PyEngine* currentEngineChecked(); +// @return borrowed ref +PyObject* getGlobalDict(); } // namespace script::py_backend diff --git a/backend/Python/PyHelper.hpp b/backend/Python/PyHelper.hpp index c3ef7e21..da1b8173 100644 --- a/backend/Python/PyHelper.hpp +++ b/backend/Python/PyHelper.hpp @@ -18,30 +18,34 @@ #pragma once #include "../../src/Native.hpp" #include "../../src/Reference.h" -#include "PyEngine.h" #include "PyHelper.h" namespace script { +class PyEngine; + struct py_interop { + // @return new ref. + template + static Local toLocal(PyObject* ref) { + return Local(Py_NewRef(ref)); + } + + // @return borrowed ref. template - static Local makeLocal(PyObject* ref) { + static Local asLocal(PyObject* ref) { return Local(ref); } - /** - * @return stolen ref. - */ + // @return new ref. template - static PyObject* toPy(const Local& ref) { - return Py_XNewRef(ref.val_); + static PyObject* getPy(const Local& ref) { + return Py_NewRef(ref.val_); } - /** - * @return borrowed ref. - */ + // @return borrowed ref. template - static PyObject* asPy(const Local& ref) { + static PyObject* peekPy(const Local& ref) { return ref.val_; } @@ -50,4 +54,31 @@ struct py_interop { } }; -} // namespace script \ No newline at end of file +namespace py_backend { + +template +class TssStorage { + private: + Py_tss_t key = Py_tss_NEEDS_INIT; + + public: + TssStorage() { + int result = PyThread_tss_create(&key); // TODO: Output or throw exception if failed + } + ~TssStorage() { + if (isValid()) PyThread_tss_delete(&key); + } + int set(T* value) { return isValid() ? PyThread_tss_set(&key, (void*)value) : 1; } + T* get() { return isValid() ? (T*)PyThread_tss_get(&key) : NULL; } + bool isValid() { return PyThread_tss_is_created(&key) > 0; } +}; + +// @return new reference +PyTypeObject* makeStaticPropertyType(); +// @return new reference +PyTypeObject* makeNamespaceType(); +// @return new reference +PyTypeObject* makeDefaultMetaclass(); + +} // namespace py_backend +} // namespace script diff --git a/backend/Python/PyLocalReference.cc b/backend/Python/PyLocalReference.cc index 4bac19aa..191c19c9 100644 --- a/backend/Python/PyLocalReference.cc +++ b/backend/Python/PyLocalReference.cc @@ -34,11 +34,11 @@ void valueConstructorCheck(PyObject* value) { } // namespace py_backend #define REF_IMPL_BASIC_FUNC(ValueType) \ - Local::Local(const Local& copy) : val_(py_backend::incRef(copy.val_)) {} \ + Local::Local(const Local& copy) : val_(Py_NewRef(copy.val_)) {} \ Local::Local(Local&& move) noexcept : val_(move.val_) { \ move.val_ = nullptr; \ } \ - Local::~Local() { py_backend::decRef(val_); } \ + Local::~Local() { Py_XDECREF(val_); } \ Local& Local::operator=(const Local& from) { \ Local(from).swap(*this); \ return *this; \ @@ -55,14 +55,14 @@ void valueConstructorCheck(PyObject* value) { } #define REF_IMPL_BASIC_NOT_VALUE(ValueType) \ - Local::Local(InternalLocalRef val) : val_(val) { \ + Local::Local(InternalLocalRef val) : val_(Py_NewRef(val)) { \ py_backend::valueConstructorCheck(val); \ } \ Local Local::describe() const { return asValue().describe(); } \ std::string Local::describeUtf8() const { return asValue().describeUtf8(); } #define REF_IMPL_TO_VALUE(ValueType) \ - Local Local::asValue() const { return Local(py_backend::incRef(val_)); } + Local Local::asValue() const { return Local(Py_NewRef(val_)); } REF_IMPL_BASIC_FUNC(Value) @@ -108,14 +108,16 @@ REF_IMPL_TO_VALUE(Unsupported) // ==== value ==== -Local::Local() noexcept : val_() {} +Local::Local() noexcept : val_(Py_NewRef(Py_None)) {} -Local::Local(InternalLocalRef ref) : val_(ref) {} +Local::Local(InternalLocalRef ref) : val_(ref) { + if (ref == nullptr) throw Exception("Python exception occurred!"); +} -bool Local::isNull() const { return val_ == nullptr; } +bool Local::isNull() const { return Py_IsNone(val_); } void Local::reset() { - py_backend::decRef(val_); + Py_DECREF(val_); val_ = nullptr; } @@ -141,127 +143,163 @@ ValueKind Local::getKind() const { } } -bool Local::isString() const { return false; } +bool Local::isString() const { return PyUnicode_CheckExact(val_); } -bool Local::isNumber() const { return PyNumber_Check(val_); } +bool Local::isNumber() const { return PyLong_CheckExact(val_) || PyFloat_CheckExact(val_); } bool Local::isBoolean() const { return PyBool_Check(val_); } -bool Local::isFunction() const { return PyCallable_Check(val_); } +bool Local::isFunction() const { return PyFunction_Check(val_) || PyCFunction_CheckExact(val_); } -bool Local::isArray() const { return false; } +bool Local::isArray() const { return PyList_CheckExact(val_); } -bool Local::isByteBuffer() const { return false; } +bool Local::isByteBuffer() const { return PyBytes_CheckExact(val_); } -bool Local::isObject() const { return false; } +// Object can be dict or class +bool Local::isObject() const { return PyDict_CheckExact(val_) || PyType_CheckExact(val_); } -bool Local::isUnsupported() const { return false; } +bool Local::isUnsupported() const { return getKind() == ValueKind::kUnsupported; } -Local Local::asString() const { throw Exception("can't cast value as String"); } +Local Local::asString() const { + if (isString()) return Local(val_); + throw Exception("can't cast value as String"); +} -Local Local::asNumber() const { throw Exception("can't cast value as Number"); } +Local Local::asNumber() const { + if (isNumber()) return Local(val_); + throw Exception("can't cast value as Number"); +} -Local Local::asBoolean() const { throw Exception("can't cast value as Boolean"); } +Local Local::asBoolean() const { + if (isBoolean()) return Local(val_); + throw Exception("can't cast value as Boolean"); +} Local Local::asFunction() const { + if (isFunction()) return Local(val_); throw Exception("can't cast value as Function"); } -Local Local::asArray() const { throw Exception("can't cast value as Array"); } +Local Local::asArray() const { + if (isArray()) return Local(val_); + throw Exception("can't cast value as Array"); +} Local Local::asByteBuffer() const { + if (isByteBuffer()) return Local(val_); throw Exception("can't cast value as ByteBuffer"); } -Local Local::asObject() const { throw Exception("can't cast value as Object"); } +Local Local::asObject() const { + if (isObject()) return Local(val_); + throw Exception("can't cast value as Object"); +} Local Local::asUnsupported() const { + if (isUnsupported()) return Local(val_); throw Exception("can't cast value as Unsupported"); } bool Local::operator==(const script::Local& other) const { - // TODO: nullptr vs None - auto lhs = val_; - auto rhs = other.val_; - - // nullptr == nullptr - if (lhs == nullptr || rhs == nullptr) { - return lhs == rhs; - } - - return PyObject_RichCompareBool(lhs, rhs, Py_EQ); + return PyObject_RichCompareBool(val_, other.val_, Py_EQ); } -Local Local::describe() const { TEMPLATE_NOT_IMPLEMENTED(); } +Local Local::describe() const { return Local(PyObject_Str(val_)); } -Local Local::get(const script::Local& key) const { return {}; } +Local Local::get(const script::Local& key) const { + PyObject* item = PyDict_GetItem(val_, key.val_); + if (item) + return py_interop::toLocal(item); + else + return Local(); +} void Local::set(const script::Local& key, - const script::Local& value) const {} + const script::Local& value) const { + PyDict_SetItem(val_, key.val_, value.val_); +} + +void Local::remove(const Local& key) const { + PyDict_DelItem(val_, key.val_); +} -void Local::remove(const Local& key) const {} -bool Local::has(const Local& key) const { return true; } +bool Local::has(const Local& key) const { + return PyDict_Contains(val_, key.val_); +} -bool Local::instanceOf(const Local& type) const { return false; } +bool Local::instanceOf(const Local& type) const { + return PyObject_IsInstance(val_, type.val_); +} -std::vector> Local::getKeys() const { return {}; } +std::vector> Local::getKeys() const { + std::vector> keys; + PyObject* key; + PyObject* value; + Py_ssize_t pos = 0; + while (PyDict_Next(val_, &pos, &key, &value)) { + keys.push_back(Local(key)); + } + return keys; +} float Local::toFloat() const { return static_cast(toDouble()); } -double Local::toDouble() const { return 0; } +double Local::toDouble() const { return PyFloat_AsDouble(val_); } int32_t Local::toInt32() const { return static_cast(toDouble()); } int64_t Local::toInt64() const { return static_cast(toDouble()); } -bool Local::value() const { return false; } +bool Local::value() const { return Py_IsTrue(val_); } Local Local::callImpl(const Local& thiz, size_t size, const Local* args) const { - // PyObject* self = thiz.isObject() ? py_interop::toPy(thiz) : nullptr; - // TODO: self - PyObject* ret = nullptr; - // args to tuple - if (size == 0) { - ret = PyObject_CallNoArgs(py_interop::asPy(*this)); - } else if (size == 1) { - ret = PyObject_CallOneArg(py_interop::asPy(*this), py_interop::asPy(args[0])); - } else { - auto tuple = PyTuple_New(static_cast(size)); - py_backend::checkException(); - for (size_t i = 0; i < size; ++i) { - PyTuple_SetItem(tuple, static_cast(i), py_interop::toPy(args[i])); - py_backend::checkException(); - } - ret = PyObject_Call(py_interop::asPy(*this), tuple, nullptr); + PyObject* args_tuple = PyTuple_New(size); + for (size_t i = 0; i < size; ++i) { + PyTuple_SetItem(args_tuple, i, args[i].val_); } - - py_backend::checkException(); - return Local(ret); + PyObject* result = PyObject_CallObject(val_, args_tuple); + Py_DECREF(args_tuple); + return py_interop::asLocal(result); } -size_t Local::size() const { return 0; } +size_t Local::size() const { return PyList_Size(val_); } -Local Local::get(size_t index) const { return {}; } +Local Local::get(size_t index) const { + PyObject* item = PyList_GetItem(val_, index); + if (item) + return py_interop::toLocal(item); + else + return Local(); +} -void Local::set(size_t index, const script::Local& value) const {} +void Local::set(size_t index, const script::Local& value) const { + size_t listSize = size(); + if (index >= listSize) + for (size_t i = listSize; i <= index; ++i) { + PyList_Append(val_, Py_None); + } + PyList_SetItem(val_, index, py_interop::getPy(value)); +} -void Local::add(const script::Local& value) const { set(size(), value); } +void Local::add(const script::Local& value) const { + PyList_Append(val_, py_interop::peekPy(value)); +} -void Local::clear() const {} +void Local::clear() const { PyList_SetSlice(val_, 0, PyList_Size(val_), nullptr); } -ByteBuffer::Type Local::getType() const { return ByteBuffer::Type::KFloat32; } +ByteBuffer::Type Local::getType() const { return ByteBuffer::Type::kInt8; } -bool Local::isShared() const { return true; } +bool Local::isShared() const { return false; } void Local::commit() const {} void Local::sync() const {} -size_t Local::byteLength() const { return 0; } +size_t Local::byteLength() const { return PyBytes_Size(val_); } -void* Local::getRawBytes() const { return nullptr; } +void* Local::getRawBytes() const { return PyBytes_AsString(val_); } -std::shared_ptr Local::getRawBytesShared() const { return {}; } +std::shared_ptr Local::getRawBytesShared() const { return nullptr; } } // namespace script diff --git a/backend/Python/PyNative.cc b/backend/Python/PyNative.cc index 703a43c0..f35a5db8 100644 --- a/backend/Python/PyNative.cc +++ b/backend/Python/PyNative.cc @@ -25,37 +25,35 @@ Arguments::Arguments(InternalCallbackInfoType callbackInfo) : callbackInfo_(call Arguments::~Arguments() = default; -Local Arguments::thiz() const { - return py_interop::makeLocal(callbackInfo_.self).asObject(); -} +Local Arguments::thiz() const { return py_interop::toLocal(callbackInfo_.self); } -bool Arguments::hasThiz() const { return callbackInfo_.self != nullptr; } +bool Arguments::hasThiz() const { return callbackInfo_.self; } -size_t Arguments::size() const { - if (!callbackInfo_.args) { - return 0; - } - return PyTuple_Size(callbackInfo_.args); -} +size_t Arguments::size() const { return PyTuple_Size(callbackInfo_.args); } Local Arguments::operator[](size_t i) const { - if (i < size()) { - return py_interop::makeLocal(PyTuple_GetItem(callbackInfo_.args, i)); + if (i > size()) { + return Local(); + } else { + return py_interop::toLocal(PyTuple_GetItem(callbackInfo_.args, i)); } - return {}; } ScriptEngine* Arguments::engine() const { return callbackInfo_.engine; } -ScriptClass::ScriptClass(const script::Local& scriptObject) : internalState_() { - TEMPLATE_NOT_IMPLEMENTED(); +ScriptClass::ScriptClass(const Local& scriptObject) : internalState_() { + internalState_.engine = py_backend::currentEngineChecked(); } -Local ScriptClass::getScriptObject() const { TEMPLATE_NOT_IMPLEMENTED(); } +Local ScriptClass::getScriptObject() const { + return py_interop::toLocal(internalState_.script_obj); +} -Local ScriptClass::getInternalStore() const { TEMPLATE_NOT_IMPLEMENTED(); } +Local ScriptClass::getInternalStore() const { + return py_interop::toLocal(internalState_.storage); +} -ScriptEngine* ScriptClass::getScriptEngine() const { TEMPLATE_NOT_IMPLEMENTED(); } +ScriptEngine* ScriptClass::getScriptEngine() const { return internalState_.engine; } -ScriptClass::~ScriptClass() = default; +ScriptClass::~ScriptClass(){}; } // namespace script \ No newline at end of file diff --git a/backend/Python/PyNative.hpp b/backend/Python/PyNative.hpp index ec9c2626..88b7f86d 100644 --- a/backend/Python/PyNative.hpp +++ b/backend/Python/PyNative.hpp @@ -27,7 +27,7 @@ ScriptClass::ScriptClass(const ScriptClass::ConstructFromCpp) : internalState template T* Arguments::engineAs() const { - return nullptr; + return static_cast(engine()); } } // namespace script \ No newline at end of file diff --git a/backend/Python/PyReference.hpp b/backend/Python/PyReference.hpp index 393788a4..bd8ea105 100644 --- a/backend/Python/PyReference.hpp +++ b/backend/Python/PyReference.hpp @@ -17,31 +17,24 @@ #pragma once #include +#include "PyHelper.hpp" namespace script { -namespace py_backend { - -inline PyObject* incRef(PyObject* ref) { return Py_XNewRef(ref); } - -inline void decRef(PyObject* ref) { Py_XDECREF(ref); } - -} // namespace py_backend - template -Global::Global() noexcept : val_() {} +Global::Global() noexcept : val_(Py_NewRef(Py_None)) {} template -Global::Global(const script::Local& localReference) {} +Global::Global(const script::Local& localReference) : val_(Py_NewRef(localReference.val_)) {} template -Global::Global(const script::Weak& weak) {} +Global::Global(const script::Weak& weak) : val_(Py_NewRef(weak.val_)) {} template -Global::Global(const script::Global& copy) : val_(copy.val_) {} +Global::Global(const script::Global& copy) : val_(Py_NewRef(copy.val_)) {} template -Global::Global(script::Global&& move) noexcept : val_(move.val_) {} +Global::Global(script::Global&& move) noexcept : val_(std::move(move.val_)) {} template Global::~Global() {} @@ -59,7 +52,9 @@ Global& Global::operator=(script::Global&& move) noexcept { } template -void Global::swap(Global& rhs) noexcept {} +void Global::swap(Global& rhs) noexcept { + std::swap(val_, rhs.val_); +} template Global& Global::operator=(const script::Local& assign) { @@ -69,35 +64,40 @@ Global& Global::operator=(const script::Local& assign) { template Local Global::get() const { - TEMPLATE_NOT_IMPLEMENTED(); + return py_interop::toLocal(val_); } template Local Global::getValue() const { - TEMPLATE_NOT_IMPLEMENTED(); + return py_interop::toLocal(val_); } template bool Global::isEmpty() const { - return false; + return val_ == nullptr; } template -void Global::reset() {} +void Global::reset() { + Py_XDECREF(val_); + val_ = nullptr; +} // == Weak == template -Weak::Weak() noexcept : val_() {} +Weak::Weak() noexcept : val_(Py_None) {} template -Weak::~Weak() {} +Weak::~Weak() { + val_ = nullptr; +} template -Weak::Weak(const script::Local& localReference) {} +Weak::Weak(const script::Local& localReference) : val_(localReference.val_) {} template -Weak::Weak(const script::Global& globalReference) {} +Weak::Weak(const script::Global& globalReference) : val_(globalReference.val_) {} template Weak::Weak(const script::Weak& copy) : val_(copy.val_) {} @@ -131,20 +131,23 @@ Weak& Weak::operator=(const script::Local& assign) { template Local Weak::get() const { if (isEmpty()) throw Exception("get on empty Weak"); - TEMPLATE_NOT_IMPLEMENTED(); + return py_interop::toLocal(val_); } template Local Weak::getValue() const { - TEMPLATE_NOT_IMPLEMENTED(); + if (isEmpty()) throw Exception("getValue on empty Weak"); + return py_interop::toLocal(val_); } template bool Weak::isEmpty() const { - return false; + return val_ == nullptr; } template -void Weak::reset() noexcept {} +void Weak::reset() noexcept { + val_ = nullptr; +} } // namespace script diff --git a/backend/Python/PyScope.cc b/backend/Python/PyScope.cc index fd45effb..527c0d8e 100644 --- a/backend/Python/PyScope.cc +++ b/backend/Python/PyScope.cc @@ -16,16 +16,96 @@ */ #include "PyScope.h" +#include "PyEngine.h" +#include -// reference +// Reference // https://docs.python.org/3.8/c-api/init.html#thread-state-and-the-global-interpreter-lock +// https://stackoverflow.com/questions/26061298/python-multi-thread-multi-interpreter-c-api +// https://stackoverflow.com/questions/15470367/pyeval-initthreads-in-python-3-how-when-to-call-it-the-saga-continues-ad-naus +// +// Because python's bad support of sub-interpreter, here to manage GIL & thread state manually. +// - One engine owns a sub-interpreter, and owns a TLS storage called engine.subThreadState_, +// which stores his own current thread state on each thread. +// - This "thread state" works like "CPU Context" in low-level C programs. When changing engine, +// "context" need to be changed to his correct thread state +// - When entering a new EngineScope, first check that if an thread state exists. If found, +// save it into oldThreadStateStack. When exit this EngineScope, old thread state saved before +// will be poped and recovered. +// - GIL is locked when any EngineScope is entered, and it is a global state (which means that +// this lock is shared by all threads). When the last EngineScope exited, the GIL will be released. +// +// GIL keeps at one time only one engine can be running and this fucking situation is caused by +// bad design of CPython. Hope that GIL will be removed in next versions and sub-interpreter support +// will be public. Only that can save us from managing these annoying things manually +// namespace script::py_backend { -EngineScopeImpl::EngineScopeImpl(PyEngine &, PyEngine *) : gilState_(PyGILState_Ensure()) {} -EngineScopeImpl::~EngineScopeImpl() { PyGILState_Release(gilState_); } +EngineScopeImpl::EngineScopeImpl(PyEngine &engine, PyEngine * enginePtr) { + managedEngine = enginePtr; + // Get thread state to enter + PyThreadState *currentThreadState = engine.subThreadState_.get(); + if (currentThreadState == NULL) { + // New thread entered first time with no threadstate + // Create a new thread state for the the sub interpreter in the new thread + // correct thread state after this + currentThreadState = PyThreadState_New(engine.subInterpreterState_); + // Save to TLS storage + engine.subThreadState_.set(currentThreadState); + } + else + { + // Thread state of this engine on current thread is inited & saved in TLS + // Check if there is another existing thread state (is another engine entered) + + // PyThreadState_GET will cause FATEL error if oldState is NULL + // so here get & check oldState by swap twice + PyThreadState* oldState = PyThreadState_Swap(NULL); + bool isOldStateNotEmpty = oldState != nullptr; + PyThreadState_Swap(oldState); + if (isOldStateNotEmpty) { + // Another engine is entered + // Push his thread state into stack + engine.oldThreadStateStack_.push(PyThreadState_Swap(NULL)); + } + // Swap to thread state of engine which is to enter + PyThreadState_Swap(currentThreadState); + } -ExitEngineScopeImpl::ExitEngineScopeImpl(PyEngine &) : threadState(PyEval_SaveThread()) {} -ExitEngineScopeImpl::~ExitEngineScopeImpl() { PyEval_RestoreThread(threadState); } + // First enginescope to enter, so lock GIL + if (PyEngine::engineEnterCount_ == 0) + { + PyEval_AcquireLock(); + } + ++PyEngine::engineEnterCount_; + // GIL locked & correct thread state here + // GIL will keep locked until last EngineScope exit +} + +EngineScopeImpl::~EngineScopeImpl() { + PyEngine *currentEngine = py_backend::currentEngine(); + if (currentEngine == managedEngine) { + // Engine existing. Need to exit + ExitEngineScopeImpl exitEngine(*currentEngine); + } +} + +ExitEngineScopeImpl::ExitEngineScopeImpl(PyEngine &engine) { + if ((--PyEngine::engineEnterCount_) == 0) + { + // This is the last enginescope to exit, so release GIL + PyEval_ReleaseLock(); + } + // Swap to clear thread state + PyThreadState_Swap(NULL); + + // Restore old thread state saved if needed + auto oldThreadStateStack = engine.oldThreadStateStack_; + if (!oldThreadStateStack.empty()) { + PyThreadState_Swap(oldThreadStateStack.top()); + oldThreadStateStack.pop(); + } +} } // namespace script::py_backend \ No newline at end of file diff --git a/backend/Python/PyScope.h b/backend/Python/PyScope.h index 0d6f98ae..9a8be6e5 100644 --- a/backend/Python/PyScope.h +++ b/backend/Python/PyScope.h @@ -24,8 +24,7 @@ namespace script::py_backend { class PyEngine; class EngineScopeImpl { - PyGILState_STATE gilState_; - + PyEngine* managedEngine; public: explicit EngineScopeImpl(PyEngine &, PyEngine *); @@ -33,12 +32,10 @@ class EngineScopeImpl { }; class ExitEngineScopeImpl { - PyThreadState *threadState; - public: explicit ExitEngineScopeImpl(PyEngine &); - ~ExitEngineScopeImpl(); + ~ExitEngineScopeImpl() = default; }; class StackFrameScopeImpl { diff --git a/backend/Python/PyUtils.cc b/backend/Python/PyUtils.cc index cd4bf063..15d3959a 100644 --- a/backend/Python/PyUtils.cc +++ b/backend/Python/PyUtils.cc @@ -19,17 +19,27 @@ namespace script { -StringHolder::StringHolder(const script::Local &string) {} +StringHolder::StringHolder(const script::Local &string) { + if (PyUnicode_Check(string.val_)) { + internalHolder_ = string.val_; + } else { + throw Exception("StringHolder require PyUnicodeObject!"); + } +} StringHolder::~StringHolder() = default; -size_t StringHolder::length() const { return 0; } +size_t StringHolder::length() const { + Py_ssize_t size = 0; + PyUnicode_AsUTF8AndSize(internalHolder_, &size); + return (size_t)size; +} -const char *StringHolder::c_str() const { return ""; } +const char *StringHolder::c_str() const { return PyUnicode_AsUTF8(internalHolder_); } -std::string_view StringHolder::stringView() const { return {}; } +std::string_view StringHolder::stringView() const { return std::string_view(c_str(), length()); } -std::string StringHolder::string() const { return {}; } +std::string StringHolder::string() const { return std::string(c_str(), length()); } #if defined(__cpp_char8_t) // NOLINTNEXTLINE(clang-analyzer-cplusplus.InnerPointer) diff --git a/backend/Python/PyValue.cc b/backend/Python/PyValue.cc index 3c8fda2e..66c6db51 100644 --- a/backend/Python/PyValue.cc +++ b/backend/Python/PyValue.cc @@ -21,36 +21,26 @@ #include "../../src/Value.h" #include "PyHelper.hpp" -using script::py_interop; -using script::py_backend::checkException; - namespace script { -template -Local checkAndMakeLocal(PyObject* ref) { - return py_interop::makeLocal(checkException(ref)); -} - -// for python this creates an empty dict -Local Object::newObject() { return checkAndMakeLocal(PyDict_New()); } +Local Object::newObject() { return py_interop::asLocal(PyDict_New()); } Local Object::newObjectImpl(const Local& type, size_t size, const Local* args) { - TEMPLATE_NOT_IMPLEMENTED(); + throw Exception("Python can't use this function"); + return Local(PyDict_New()); } Local String::newString(const char* utf8) { - return checkAndMakeLocal(PyBytes_FromString(utf8)); + return py_interop::asLocal(PyUnicode_FromString(utf8)); } Local String::newString(std::string_view utf8) { - return checkAndMakeLocal( - PyBytes_FromStringAndSize(utf8.data(), static_cast(utf8.length()))); + return py_interop::asLocal(PyUnicode_FromStringAndSize(utf8.data(), utf8.size())); } Local String::newString(const std::string& utf8) { - return checkAndMakeLocal( - PyBytes_FromStringAndSize(utf8.c_str(), static_cast(utf8.length()))); + return newString(std::string_view(utf8)); } #if defined(__cpp_char8_t) @@ -63,94 +53,93 @@ Local String::newString(std::u8string_view utf8) { return newString(std::string_view(reinterpret_cast(utf8.data()), utf8.length())); } -Local String::newString(const std::u8string& utf8) { return newString(utf8.c_str()); } +Local String::newString(const std::u8string& utf8) { + return newString(std::u8string_view(utf8)); +} #endif Local Number::newNumber(float value) { return newNumber(static_cast(value)); } Local Number::newNumber(double value) { - return checkAndMakeLocal(PyLong_FromDouble(value)); + return py_interop::asLocal(PyFloat_FromDouble(value)); } Local Number::newNumber(int32_t value) { - return checkAndMakeLocal(PyLong_FromLong(static_cast(value))); + return py_interop::asLocal(PyLong_FromLong(value)); } Local Number::newNumber(int64_t value) { - return checkAndMakeLocal(PyLong_FromLongLong(static_cast(value))); + return py_interop::asLocal(PyLong_FromLongLong(value)); } Local Boolean::newBoolean(bool value) { - return checkAndMakeLocal(PyBool_FromLong(value)); -} - -namespace { - -static constexpr const char* kFunctionDataName = "_ScriptX_function_data"; - -struct FunctionData { - FunctionCallback function; - py_backend::PyEngine* engine = nullptr; -}; - -} // namespace - -Local Function::newFunction(script::FunctionCallback callback) { - auto callbackIns = std::make_unique(); - callbackIns->engine = EngineScope::currentEngineAs(); - callbackIns->function = std::move(callback); - - PyMethodDef method{}; - method.ml_name = "ScriptX_native_method"; - method.ml_flags = METH_O; - method.ml_doc = "ScriptX Function::newFunction"; - method.ml_meth = [](PyObject* self, PyObject* args) -> PyObject* { - auto ptr = PyCapsule_GetPointer(self, kFunctionDataName); - if (ptr == nullptr) { - ::PyErr_SetString(PyExc_TypeError, "invalid 'self' for native method"); - } else { - auto data = static_cast(ptr); - try { - auto ret = data->function(py_interop::makeArguments(nullptr, self, args)); - return py_interop::toPy(ret); - } catch (Exception& e) { - py_backend::rethrowException(e); - } + return py_interop::asLocal(PyBool_FromLong(value)); +} + +Local Function::newFunction(FunctionCallback callback) { + struct FunctionData { + FunctionCallback function; + py_backend::PyEngine* engine; + }; + + PyMethodDef* method = new PyMethodDef; + method->ml_name = "scriptx_function"; + method->ml_flags = METH_VARARGS; + method->ml_doc = nullptr; + method->ml_meth = [](PyObject* self, PyObject* args) -> PyObject* { + auto data = static_cast(PyCapsule_GetPointer(self, nullptr)); + try { + return py_interop::peekPy( + data->function(py_interop::makeArguments(data->engine, self, args))); + } catch (const Exception& e) { + py_backend::rethrowException(e); } return nullptr; }; - auto ctx = PyCapsule_New(callbackIns.get(), kFunctionDataName, [](PyObject* cap) { - auto ptr = PyCapsule_GetPointer(cap, kFunctionDataName); + PyCapsule_Destructor destructor = [](PyObject* cap) { + void* ptr = PyCapsule_GetPointer(cap, nullptr); delete static_cast(ptr); - }); - py_backend::checkException(ctx); - callbackIns.release(); + }; + PyObject* capsule = PyCapsule_New( + new FunctionData{std::move(callback), py_backend::currentEngine()}, nullptr, destructor); + py_backend::checkPyErr(); - PyObject* closure = PyCFunction_New(&method, ctx); - Py_XDECREF(ctx); - py_backend::checkException(closure); + PyObject* function = PyCFunction_New(method, capsule); + Py_DECREF(capsule); + py_backend::checkPyErr(); - return Local(closure); + return py_interop::asLocal(function); } -Local Array::newArray(size_t size) { - return checkAndMakeLocal(PyList_New(static_cast(size))); -} +Local Array::newArray(size_t size) { return py_interop::asLocal(PyList_New(size)); } Local Array::newArrayImpl(size_t size, const Local* args) { - TEMPLATE_NOT_IMPLEMENTED(); + PyObject* list = PyList_New(size); + if (!list) { + throw Exception(); + } + for (size_t i = 0; i < size; ++i) { + PyList_SetItem(list, i, py_interop::getPy(args[i])); + } + return py_interop::asLocal(list); } -Local ByteBuffer::newByteBuffer(size_t size) { TEMPLATE_NOT_IMPLEMENTED(); } +Local ByteBuffer::newByteBuffer(size_t size) { + const char* bytes = new char[size]{}; + PyObject* result = PyBytes_FromStringAndSize(bytes, size); + delete bytes; + return py_interop::asLocal(result); +} Local ByteBuffer::newByteBuffer(void* nativeBuffer, size_t size) { - TEMPLATE_NOT_IMPLEMENTED(); + return py_interop::asLocal( + PyBytes_FromStringAndSize(static_cast(nativeBuffer), size)); } Local ByteBuffer::newByteBuffer(std::shared_ptr nativeBuffer, size_t size) { - TEMPLATE_NOT_IMPLEMENTED(); + return newByteBuffer(nativeBuffer.get(), size); } } // namespace script \ No newline at end of file diff --git a/backend/Python/trait/TraitException.h b/backend/Python/trait/TraitException.h index 80b13f49..a5efc9e3 100644 --- a/backend/Python/trait/TraitException.h +++ b/backend/Python/trait/TraitException.h @@ -25,7 +25,16 @@ namespace py_backend { class ExceptionFields { public: + mutable Global exception_{}; //exception capsule + mutable std::string message_{}; + mutable bool hasMessage_ = false; + + mutable std::string stacktrace_{}; + mutable bool hasStacktrace_ = false; + + void fillMessage() const noexcept; + void fillStacktrace() const noexcept; }; } // namespace py_backend diff --git a/backend/Python/trait/TraitNative.h b/backend/Python/trait/TraitNative.h index 3022f038..f7cf6f4f 100644 --- a/backend/Python/trait/TraitNative.h +++ b/backend/Python/trait/TraitNative.h @@ -24,14 +24,15 @@ namespace script { namespace py_backend { struct ArgumentsData { - mutable PyEngine* engine; + PyEngine* engine; PyObject* self; PyObject* args; }; struct ScriptClassState { - ScriptEngine* scriptEngine_ = nullptr; - Weak weakRef_; + ScriptEngine* engine = nullptr; + PyObject* script_obj; + PyObject* storage; }; } // namespace py_backend @@ -46,4 +47,4 @@ struct internal::ImplType<::script::ScriptClass> { using type = py_backend::ScriptClassState; }; -} // namespace script \ No newline at end of file +} // namespace script diff --git a/backend/Python/trait/TraitReference.h b/backend/Python/trait/TraitReference.h index d45c7ed5..95379939 100644 --- a/backend/Python/trait/TraitReference.h +++ b/backend/Python/trait/TraitReference.h @@ -18,7 +18,6 @@ #pragma once #include -#include #include "../../src/types.h" #include "../PyHelper.h" diff --git a/backend/Python/trait/TraitUtils.h b/backend/Python/trait/TraitUtils.h index 215f6802..1f0bc116 100644 --- a/backend/Python/trait/TraitUtils.h +++ b/backend/Python/trait/TraitUtils.h @@ -17,6 +17,7 @@ #pragma once #include "../../src/types.h" +#include "../PyHelper.h" namespace script { @@ -24,7 +25,7 @@ struct py_interop; template <> struct internal::ImplType { - using type = int; + using type = PyObject*; }; template <> diff --git a/backend/QuickJs/QjsEngine.cc b/backend/QuickJs/QjsEngine.cc index e71a6e7d..3a46e789 100644 --- a/backend/QuickJs/QjsEngine.cc +++ b/backend/QuickJs/QjsEngine.cc @@ -17,6 +17,7 @@ #include "QjsEngine.h" #include +#include "../../src/utils/Helper.hpp" namespace script::qjs_backend { @@ -268,6 +269,27 @@ Local QjsEngine::eval(const Local& script, const Local& so return Local(ret); } +Local QjsEngine::loadFile(const Local& scriptFile) { + if(scriptFile.toString().empty()) + throw Exception("script file no found"); + Local content = internal::readAllFileContent(scriptFile); + if(content.isNull()) + throw Exception("can't load script file"); + + std::string sourceFilePath = scriptFile.toString(); + std::size_t pathSymbol = sourceFilePath.rfind("/"); + if(pathSymbol != -1) + sourceFilePath = sourceFilePath.substr(pathSymbol + 1); + else + { + pathSymbol = sourceFilePath.rfind("\\"); + if(pathSymbol != -1) + sourceFilePath = sourceFilePath.substr(pathSymbol + 1); + } + Local sourceFileName = String::newString(sourceFilePath); + return eval(content.asString(), sourceFileName); +} + std::shared_ptr QjsEngine::messageQueue() { return queue_; } void QjsEngine::gc() { diff --git a/backend/QuickJs/QjsEngine.h b/backend/QuickJs/QjsEngine.h index 6ea4966a..b810323d 100644 --- a/backend/QuickJs/QjsEngine.h +++ b/backend/QuickJs/QjsEngine.h @@ -94,6 +94,8 @@ class QjsEngine : public ScriptEngine { Local eval(const Local& script) override; using ScriptEngine::eval; + Local loadFile(const Local& scriptFile) override; + std::shared_ptr messageQueue() override; void gc() override; diff --git a/backend/QuickJs/QjsLocalReference.cc b/backend/QuickJs/QjsLocalReference.cc index 5e154e0b..d0a72808 100644 --- a/backend/QuickJs/QjsLocalReference.cc +++ b/backend/QuickJs/QjsLocalReference.cc @@ -209,7 +209,7 @@ bool Local::operator==(const script::Local& other) const { auto context = engine.context_; #ifdef QUICK_JS_HAS_SCRIPTX_PATCH return JS_StrictEqual(context, val_, qjs_interop::peekLocal(other)); -#elif +#else auto fun = qjs_interop::makeLocal( qjs_backend::dupValue(engine.helperFunctionStrictEqual_, context)); return fun.call({}, *this, other).asBoolean().value(); diff --git a/backend/V8/V8Engine.cc b/backend/V8/V8Engine.cc index 23686521..a208ac06 100644 --- a/backend/V8/V8Engine.cc +++ b/backend/V8/V8Engine.cc @@ -19,6 +19,7 @@ #include #include #include +#include "../../src/utils/Helper.hpp" namespace script::v8_backend { @@ -174,6 +175,27 @@ Local V8Engine::eval(const Local& script, const Local& so Local V8Engine::eval(const Local& script) { return eval(script, {}); } +Local V8Engine::loadFile(const Local& scriptFile) { + if(scriptFile.toString().empty()) + throw Exception("script file no found"); + Local content = internal::readAllFileContent(scriptFile); + if(content.isNull()) + throw Exception("can't load script file"); + + std::string sourceFilePath = scriptFile.toString(); + std::size_t pathSymbol = sourceFilePath.rfind("/"); + if(pathSymbol != -1) + sourceFilePath = sourceFilePath.substr(pathSymbol + 1); + else + { + pathSymbol = sourceFilePath.rfind("\\"); + if(pathSymbol != -1) + sourceFilePath = sourceFilePath.substr(pathSymbol + 1); + } + Local sourceFileName = String::newString(sourceFilePath); + return eval(content.asString(), sourceFileName); +} + void V8Engine::registerNativeClassStatic(v8::Local funcT, const internal::StaticDefine* staticDefine) { for (auto& prop : staticDefine->properties) { diff --git a/backend/V8/V8Engine.h b/backend/V8/V8Engine.h index 8942ecfa..ad9b1375 100644 --- a/backend/V8/V8Engine.h +++ b/backend/V8/V8Engine.h @@ -115,6 +115,8 @@ class V8Engine : public ::script::ScriptEngine { Local eval(const Local& script) override; using ScriptEngine::eval; + Local loadFile(const Local& scriptFile) override; + /** * Create a new V8 Engine that share the same isolate, but with different context. * Caller own the returned pointer, and the returned instance diff --git a/backend/WebAssembly/WasmEngine.cc b/backend/WebAssembly/WasmEngine.cc index c52d446b..727323da 100644 --- a/backend/WebAssembly/WasmEngine.cc +++ b/backend/WebAssembly/WasmEngine.cc @@ -23,6 +23,7 @@ #include "WasmNative.hpp" #include "WasmReference.hpp" #include "WasmScope.hpp" +#include "../../src/utils/Helper.hpp" namespace script::wasm_backend { @@ -76,6 +77,27 @@ Local WasmEngine::eval(const Local& script, const Local& s return Local(retIndex); } +Local WasmEngine::loadFile(const Local& scriptFile) { + if(scriptFile.toString().empty()) + throw Exception("script file no found"); + Local content = internal::readAllFileContent(scriptFile); + if(content.isNull()) + throw Exception("can't load script file"); + + std::string sourceFilePath = scriptFile.toString(); + std::size_t pathSymbol = sourceFilePath.rfind("/"); + if(pathSymbol != -1) + sourceFilePath = sourceFilePath.substr(pathSymbol + 1); + else + { + pathSymbol = sourceFilePath.rfind("\\"); + if(pathSymbol != -1) + sourceFilePath = sourceFilePath.substr(pathSymbol + 1); + } + Local sourceFileName = String::newString(sourceFilePath); + return eval(content.asString(), sourceFileName); +} + std::shared_ptr WasmEngine::messageQueue() { return messageQueue_; } void WasmEngine::gc() {} diff --git a/backend/WebAssembly/WasmEngine.h b/backend/WebAssembly/WasmEngine.h index b8d629f2..9384dcba 100644 --- a/backend/WebAssembly/WasmEngine.h +++ b/backend/WebAssembly/WasmEngine.h @@ -74,6 +74,8 @@ class WasmEngine : public ScriptEngine { Local eval(const Local& script) override; using ScriptEngine::eval; + Local loadFile(const Local& scriptFile) override; + std::shared_ptr messageQueue() override; void gc() override; diff --git a/backend/WebAssembly/WasmHelper.cc b/backend/WebAssembly/WasmHelper.cc index 8fafa653..761653ff 100644 --- a/backend/WebAssembly/WasmHelper.cc +++ b/backend/WebAssembly/WasmHelper.cc @@ -824,7 +824,7 @@ CHECKED_EM_JS(int, _ScriptX_Native_getInternalStore, (int weakThis), { return stack.push(thiz[sym]); }); -CHECKED_EM_JS(void, _ScriptX_Native_destoryInstance, (int scriptClassInstance), { +CHECKED_EM_JS(void, _ScriptX_Native_destroyInstance, (int scriptClassInstance), { const stack = Module.SCRIPTX_STACK; scriptClassInstance = stack.values[scriptClassInstance]; @@ -1243,7 +1243,7 @@ void wasm_interop::destroySharedByteBuffer(const Local &sharedByteBu } void wasm_interop::destroyScriptClass(const Local &scriptClass) { - CHECKED_VOID_JS_CALL(wasm_backend::_ScriptX_Native_destoryInstance( + CHECKED_VOID_JS_CALL(wasm_backend::_ScriptX_Native_destroyInstance( wasm_backend::WasmEngine::refIndex(scriptClass))); } diff --git a/docs/en/Python.md b/docs/en/Python.md new file mode 100644 index 00000000..0a010cab --- /dev/null +++ b/docs/en/Python.md @@ -0,0 +1,25 @@ +# Python Language + +ScriptX and Python language type comparison table + +| Python | ScriptX | +| :--------: | :------: | +| None | Null | +| dict | Object | +| list | Array | +| string | String | +| int, float | Number | +| bool | Boolean | +| function | Function | + +## Language specific implementation of Object + +Unlike JavaScript and Lua, Python has an internal generic object base class Py_Object that cannot be instantiated (equivalent to an abstract base class), so it is not possible to fully equate the Object concepts of these two languages to Python. + +Python's Object is currently implemented using `Py_Dict`, which is analogous to Lua's table.It is normal to set & get member properties and methods using `set` and `get`, and call member methods . But you can't use `Object::newObject` to call its constructor to construct a new object of the same type -- because they're both of type dict, and there's no constructor + +## `eval` return value problem + +The Python API provides two types of interfaces for executing code: the `eval` type can only execute a single expression and return its result, while the `exec` type provides support for executing multiple lines of code, which is the normal way of reading a file to execute code, but the return value is always `None`. This is due to the special design of the Python interpreter, which differs significantly from other languages. + +Therefore, in the ScriptX implementation, if you use `Engine::eval` to execute a multi-line statement, the return value of `eval` will always be `Null`. If you need to get the return value, you can add an assignment line at the end of the executed code, and then use `Engine::get` to get the data of the result variable from the engine after `eval` finished. \ No newline at end of file diff --git a/docs/en/WebAssembly.md b/docs/en/WebAssembly.md index d020afb8..ebfcfa89 100644 --- a/docs/en/WebAssembly.md +++ b/docs/en/WebAssembly.md @@ -238,13 +238,13 @@ class SharedByteBuffer { readonly byteLength: number; // Manual memory management, destroy this class, and release WASM memory - public destory(): void; + public destroy(): void; } ``` Pay attention to the buffer attribute above. As mentioned above, when WASM `grow_memory`, the underlying `ArrayBuffer` may change, so when using `SharedByteBuffer`, create a `TypedArray` immediately, don’t keep the reference for long-term use (of course you You can configure wasm not to grow_memory, or use `SharedArrayBuffer`, so the buffer attribute will not change, depending on your usage scenario). -Finally, because WASM has no GC and JS has no finalize, the user needs to release this memory. You can use the above `destory` method, or you can use `ScriptX.destroySharedByteBuffer`. The C++ code uses `wasm_interop::destroySharedByteBuffer`. +Finally, because WASM has no GC and JS has no finalize, the user needs to release this memory. You can use the above `destroy` method, or you can use `ScriptX.destroySharedByteBuffer`. The C++ code uses `wasm_interop::destroySharedByteBuffer`. Give a chestnut: diff --git a/docs/zh/Basics.md b/docs/zh/Basics.md index a6dc07b5..5594ae44 100644 --- a/docs/zh/Basics.md +++ b/docs/zh/Basics.md @@ -149,7 +149,7 @@ void doFrame() { ### Message::tag -有一点需要注意，因为部分backend允许多个ScriptEngine共享一个MessageQueue；所以当你使用该特性时，MessageQueue的Message有一个tag字段，用来区分这个Message属于哪个ScriptEngine，因此在postMessage的时候请指定tag，这样ScriptEngine在destory的时候会把到期没执行的Message全部release掉，并调用其release handler。（通过`messageQueue.removeMessageByTag(scriptEngine)`实现。) +有一点需要注意，因为部分backend允许多个ScriptEngine共享一个MessageQueue；所以当你使用该特性时，MessageQueue的Message有一个tag字段，用来区分这个Message属于哪个ScriptEngine，因此在postMessage的时候请指定tag，这样ScriptEngine在destroy的时候会把到期没执行的Message全部release掉，并调用其release handler。（通过`messageQueue.removeMessageByTag(scriptEngine)`实现。) PS: 如果一个ScriptEngine只对应一个MessageQueue，则在ScriptEngine destroy的时候会析构掉MessageQueue，那么内部的**所有** Message 都将release，这种情况可以不设置tag字段。 diff --git a/docs/zh/Python.md b/docs/zh/Python.md new file mode 100644 index 00000000..c3a76553 --- /dev/null +++ b/docs/zh/Python.md @@ -0,0 +1,25 @@ +# Python语言 + +ScriptX和Python语言类型对照表 + +| Python | ScriptX | +| :--------: | :------: | +| None | Null | +| dict | Object | +| list | Array | +| string | String | +| int, float | Number | +| bool | Boolean | +| function | Function | + +## Object 的语言特定实现 + +和 JavaScript 与 Lua 不同，Python 在内部通用的对象基类 Py_Object 无法被实例化（相当于抽象基类），因此无法将这两种语言中的 Object 概念完全等同到 Python 中。 + +目前 Python 的 Object 使用 `Py_Dict` 实现，类比于 Lua 的 table，同样可以使用 `set` `get` 设置成员属性和方法，并调用成员方法。但是无法使用 `Object::newObject` 调用其构造函数构造一个同类型的新对象 —— 因为它们的类型都是 dict，不存在构造函数 + +## `eval` 返回值问题 + +Python API 提供的执行代码接口分为两种：其中 eval 类型的接口只能执行单个表达式，并返回其结果；exec 类型的接口对执行多行代码提供支持（也就是正常读取文件执行代码所采取的方式），但是返回值恒定为`None`。这是由于 Python 解释器特殊的设计造成，与其他语言有较大差异。 + +因此，在ScriptX的实现中，如果使用 `Engine::eval`执行多行语句，则 `eval` 返回值一定为 `Null`。如果需要获取返回值，可以在所执行的代码最后添加一行赋值，并在 `eval` 执行完毕后使用 `Engine::get` 从引擎获取结果变量的数据。 \ No newline at end of file diff --git a/docs/zh/WebAssembly.md b/docs/zh/WebAssembly.md index 7bc896f8..e4582cf4 100644 --- a/docs/zh/WebAssembly.md +++ b/docs/zh/WebAssembly.md @@ -238,13 +238,13 @@ class SharedByteBuffer { readonly byteLength: number; // 手动内存管理，销毁该类，并释放WASM的内存 - public destory(): void; + public destroy(): void; } ``` 注意上面的buffer属性，如上文所述，当WASM `grow_memory` 的时候，底层的`ArrayBuffer`可能会变，因此使用`SharedByteBuffer`的时候要即时创建`TypedArray`，不要保留引用长期使用（当然你可以配置wasm不grow_memory，或者使用`SharedArrayBuffer`，这样buffer属性一定不会变，具体还是取决于你的使用场景）。 -最后还是因为WASM 没有GC, JS 没有finalize，因此这段内存需要使用者自己去释放。可以使用上述`destory`方法，也可以使用`ScriptX.destroySharedByteBuffer`。C++代码使用`wasm_interop::destroySharedByteBuffer`。 +最后还是因为WASM 没有GC, JS 没有finalize，因此这段内存需要使用者自己去释放。可以使用上述`destroy`方法，也可以使用`ScriptX.destroySharedByteBuffer`。C++代码使用`wasm_interop::destroySharedByteBuffer`。 举个栗子： diff --git a/src/Engine.h b/src/Engine.h index 6697efb6..6cf7b665 100644 --- a/src/Engine.h +++ b/src/Engine.h @@ -120,6 +120,17 @@ class ScriptEngine { String::newString(std::forward(sourceFileStringLike))); } + /** + * @param scriptFile path of script file to load + * @return evaluate result + */ + virtual Local loadFile(const Local& scriptFile) = 0; + + template + Local loadFile(T&& scriptFileStringLike) { + return loadFile(String::newString(std::forward(scriptFileStringLike))); + } + /** * register a native class definition (constructor & property & function) to script. * @tparam T a subclass of the NativeClass, which implements all the Script-Native method in cpp. diff --git a/src/Reference.cc b/src/Reference.cc index a0bc3044..19cba191 100644 --- a/src/Reference.cc +++ b/src/Reference.cc @@ -43,4 +43,8 @@ std::vector Local::getKeyNames() const { return ret; } +bool Local::isInteger() const { + return toDouble() - toInt64() < 1e-15; +} + } // namespace script diff --git a/src/Reference.h b/src/Reference.h index a4303bc2..e20db2f1 100644 --- a/src/Reference.h +++ b/src/Reference.h @@ -430,6 +430,8 @@ class Local { double toDouble() const; + bool isInteger() const; + SPECIALIZE_NON_VALUE(Number) }; diff --git a/src/Scope.h b/src/Scope.h index 79f71aa4..8ced6fe7 100644 --- a/src/Scope.h +++ b/src/Scope.h @@ -88,7 +88,7 @@ class EngineScope final { static T* currentEngineAs() { auto currentScope = getCurrent(); if (currentScope) { - return internal::scriptDynamicCast(getCurrent()->engine_); + return internal::scriptDynamicCast(currentScope->engine_); } return nullptr; } @@ -105,7 +105,7 @@ class EngineScope final { auto currentScope = getCurrent(); if (currentScope) { - engine = internal::scriptDynamicCast(getCurrent()->engine_); + engine = internal::scriptDynamicCast(currentScope->engine_); } ensureEngineScope(engine); diff --git a/src/utils/Helper.cc b/src/utils/Helper.cc index 18eb70c4..37040f03 100644 --- a/src/utils/Helper.cc +++ b/src/utils/Helper.cc @@ -18,6 +18,7 @@ #include #include +#include namespace script::internal { @@ -56,4 +57,17 @@ Local getNamespaceObject(ScriptEngine* engine, const std::string_view& na return nameSpaceObj; } +Local readAllFileContent(const Local& scriptFile) +{ + std::ifstream fRead; + fRead.open(scriptFile.toString(), std::ios_base::in); + if (!fRead.is_open()) { + return Local(); + } + std::string data((std::istreambuf_iterator(fRead)), + std::istreambuf_iterator()); + fRead.close(); + return String::newString(std::move(data)).asValue(); +} + } // namespace script::internal \ No newline at end of file diff --git a/src/utils/Helper.hpp b/src/utils/Helper.hpp index 6790ccb3..541791dd 100644 --- a/src/utils/Helper.hpp +++ b/src/utils/Helper.hpp @@ -55,4 +55,6 @@ void withNArray(size_t N, FN&& fn) { Local getNamespaceObject(ScriptEngine* engine, const std::string_view& nameSpace, Local rootNs = {}); + +Local readAllFileContent(const Local& scriptFile); } // namespace script::internal \ No newline at end of file diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index e277e827..a22a4a1a 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -79,7 +79,7 @@ target_sources(UnitTests PRIVATE # 1. import ScriptX # set which backend engine to use -#set(SCRIPTX_BACKEND V8 CACHE STRING "" FORCE) +set(SCRIPTX_BACKEND Python CACHE STRING "" FORCE) # we want the default behavior, so don't set this # set(SCRIPTX_NO_EXCEPTION_ON_BIND_FUNCTION YES CACHE BOOL "" FORCE) diff --git a/test/cmake/TestEnv.cmake b/test/cmake/TestEnv.cmake index 37b2b30e..246bb25f 100644 --- a/test/cmake/TestEnv.cmake +++ b/test/cmake/TestEnv.cmake @@ -35,11 +35,12 @@ endif () if ("${SCRIPTX_BACKEND}" STREQUAL "") ### choose your backend - set(SCRIPTX_BACKEND V8 CACHE STRING "" FORCE) + #set(SCRIPTX_BACKEND V8 CACHE STRING "" FORCE) #set(SCRIPTX_BACKEND JavaScriptCore CACHE STRING "" FORCE) #set(SCRIPTX_BACKEND Lua CACHE STRING "" FORCE) #set(SCRIPTX_BACKEND WebAssembly CACHE STRING "" FORCE) #set(SCRIPTX_BACKEND QuickJs CACHE STRING "" FORCE) + set(SCRIPTX_BACKEND Python CACHE STRING "" FORCE) #set(SCRIPTX_BACKEND Empty CACHE STRING "" FORCE) endif () @@ -53,47 +54,47 @@ include(${CMAKE_CURRENT_LIST_DIR}/test_libs/CMakeLists.txt) if (${SCRIPTX_BACKEND} STREQUAL V8) if (SCRIPTX_TEST_BUILD_ONLY) set(DEVOPS_LIBS_INCLUDE - "${SCRIPTX_TEST_LIBS}/mac/v8/include" + "${SCRIPTX_TEST_LIBS}/v8/mac/include" CACHE STRING "" FORCE) elseif (APPLE) set(DEVOPS_LIBS_INCLUDE - "${SCRIPTX_TEST_LIBS}/mac/v8/include" + "${SCRIPTX_TEST_LIBS}/v8/mac/include" CACHE STRING "" FORCE) set(DEVOPS_LIBS_LIBPATH - "${SCRIPTX_TEST_LIBS}/mac/v8/libv8_monolith.a" + "${SCRIPTX_TEST_LIBS}/v8/mac/libv8_monolith.a" CACHE STRING "" FORCE) elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux") # v8 8.8 set(DEVOPS_LIBS_INCLUDE - "${SCRIPTX_TEST_LIBS}/linux64/v8/include" + "${SCRIPTX_TEST_LIBS}/v8/linux64/include" CACHE STRING "" FORCE) set(DEVOPS_LIBS_LIBPATH - "${SCRIPTX_TEST_LIBS}/linux64/v8/libv8_monolith.a" + "${SCRIPTX_TEST_LIBS}/v8/linux64/libv8_monolith.a" CACHE STRING "" FORCE) set(DEVOPS_LIBS_MARCO V8_COMPRESS_POINTERS CACHE STRING "" FORCE) elseif (WIN32) set(DEVOPS_LIBS_INCLUDE - "${SCRIPTX_TEST_LIBS}/win64/v8/include" + "${SCRIPTX_TEST_LIBS}/v8/win64/include" CACHE STRING "" FORCE) set(DEVOPS_LIBS_LIBPATH - "${SCRIPTX_TEST_LIBS}/win64/v8/v8_libbase.dll.lib" - "${SCRIPTX_TEST_LIBS}/win64/v8/v8_libplatform.dll.lib" - "${SCRIPTX_TEST_LIBS}/win64/v8/v8.dll.lib" + "${SCRIPTX_TEST_LIBS}/v8/win64/v8_libbase.dll.lib" + "${SCRIPTX_TEST_LIBS}/v8/win64/v8_libplatform.dll.lib" + "${SCRIPTX_TEST_LIBS}/v8/win64/v8.dll.lib" CACHE STRING "" FORCE) add_custom_command(TARGET UnitTests POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_directory - "${SCRIPTX_TEST_LIBS}/win64/v8/dll" $ + "${SCRIPTX_TEST_LIBS}/v8/win64/dll" $ ) endif () elseif (${SCRIPTX_BACKEND} STREQUAL JavaScriptCore) if (SCRIPTX_TEST_BUILD_ONLY) set(DEVOPS_LIBS_INCLUDE - "${SCRIPTX_TEST_LIBS}/win64/jsc/include" + "${SCRIPTX_TEST_LIBS}/jsc/win32/include" CACHE STRING "" FORCE) elseif (APPLE) set(DEVOPS_LIBS_INCLUDE @@ -105,14 +106,14 @@ elseif (${SCRIPTX_BACKEND} STREQUAL JavaScriptCore) CACHE STRING "" FORCE) elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux") set(DEVOPS_LIBS_INCLUDE - "${SCRIPTX_TEST_LIBS}/linux64/jsc/Headers" + "${SCRIPTX_TEST_LIBS}/jsc/linux64/Headers" CACHE STRING "" FORCE) set(DEVOPS_LIBS_LIBPATH #"-Wl,--start-group" - "${SCRIPTX_TEST_LIBS}/linux64/jsc/libJavaScriptCore.a" - "${SCRIPTX_TEST_LIBS}/linux64/jsc/libWTF.a" - "${SCRIPTX_TEST_LIBS}/linux64/jsc/libbmalloc.a" + "${SCRIPTX_TEST_LIBS}/jsc/linux64/libJavaScriptCore.a" + "${SCRIPTX_TEST_LIBS}/jsc/linux64/libWTF.a" + "${SCRIPTX_TEST_LIBS}/jsc/linux64/libbmalloc.a" "dl" "icudata" "icui18n" @@ -122,16 +123,16 @@ elseif (${SCRIPTX_BACKEND} STREQUAL JavaScriptCore) CACHE STRING "" FORCE) elseif (WIN32) set(DEVOPS_LIBS_INCLUDE - "${SCRIPTX_TEST_LIBS}/win64/jsc/include" + "${SCRIPTX_TEST_LIBS}/jsc/win32/include" CACHE STRING "" FORCE) set(DEVOPS_LIBS_LIBPATH - "${SCRIPTX_TEST_LIBS}/win64/jsc/JavaScriptCore.lib" + "${SCRIPTX_TEST_LIBS}/jsc/win32/JavaScriptCore.lib" CACHE STRING "" FORCE) add_custom_command(TARGET UnitTests POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_directory - "${SCRIPTX_TEST_LIBS}/win64/jsc/dll" $ + "${SCRIPTX_TEST_LIBS}/jsc/win32/dll" $ ) endif () @@ -146,8 +147,24 @@ elseif (${SCRIPTX_BACKEND} STREQUAL QuickJs) include("${SCRIPTX_TEST_LIBS}/quickjs/CMakeLists.txt") set(DEVOPS_LIBS_LIBPATH QuickJs CACHE STRING "" FORCE) elseif (${SCRIPTX_BACKEND} STREQUAL Python) - set(DEVOPS_LIBS_INCLUDE - "/usr/local/Cellar/python@3.10/3.10.0_2/Frameworks/Python.framework/Headers/" - CACHE STRING "" FORCE) - set(DEVOPS_LIBS_LIBPATH "/usr/local/Cellar/python@3.10/3.10.0_2/Frameworks/Python.framework/Versions/Current/lib/libpython3.10.dylib" CACHE STRING "" FORCE) + if (WIN32) + set(DEVOPS_LIBS_INCLUDE + "${SCRIPTX_TEST_LIBS}/python/win64/include" + CACHE STRING "" FORCE) + + set(DEVOPS_LIBS_LIBPATH + "${SCRIPTX_TEST_LIBS}/python/win64/python310_d.lib" + CACHE STRING "" FORCE) + + add_custom_command(TARGET UnitTests POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_directory + "${SCRIPTX_TEST_LIBS}/python/win64/dll" $ + ) + else () + set(DEVOPS_LIBS_INCLUDE + "/usr/local/Cellar/python@3.10/3.10.0_2/Frameworks/Python.framework/Headers/" + CACHE STRING "" FORCE) + set(DEVOPS_LIBS_LIBPATH "/usr/local/Cellar/python@3.10/3.10.0_2/Frameworks/Python.framework/Versions/Current/lib/libpython3.10.dylib" CACHE STRING "" FORCE) + endif () endif () + diff --git a/test/cmake/test_libs/CMakeLists.txt.in b/test/cmake/test_libs/CMakeLists.txt.in index 1a36c328..672a0042 100644 --- a/test/cmake/test_libs/CMakeLists.txt.in +++ b/test/cmake/test_libs/CMakeLists.txt.in @@ -4,7 +4,7 @@ project(ScriptXLibs-download NONE) include(ExternalProject) ExternalProject_Add(TestLibs - GIT_REPOSITORY https://github.com/LanderlYoung/ScriptXTestLibs.git + GIT_REPOSITORY https://github.com/LiteLDev/ScriptXTestLibs.git GIT_TAG ${DEPENDENCY_SCRIPTX_LIBS_BRANCH} GIT_SHALLOW 1 SOURCE_DIR "${SCRIPTX_TEST_LIBS}" diff --git a/test/src/EngineTest.cc b/test/src/EngineTest.cc index ed73711d..0081fc87 100644 --- a/test/src/EngineTest.cc +++ b/test/src/EngineTest.cc @@ -70,7 +70,7 @@ TEST_F(EngineTest, SmartPointer) { uniquePtr.release(); uniquePtr1.release(); engine = nullptr; - // destory engine + // destroy engine sharedPtr.reset(); } diff --git a/test/src/ExceptionTest.cc b/test/src/ExceptionTest.cc index d2d637ca..1c4bb884 100644 --- a/test/src/ExceptionTest.cc +++ b/test/src/ExceptionTest.cc @@ -175,7 +175,7 @@ TEST_F(ExceptionTest, Cross) { } #ifndef SCRIPTX_BACKEND_WEBASSEMBLY -TEST_F(ExceptionTest, EngineScopeOnDestory) { +TEST_F(ExceptionTest, EngineScopeOnDestroy) { bool executed = false; { EngineScope engineScope(engine); diff --git a/test/src/ManagedObjectTest.cc b/test/src/ManagedObjectTest.cc index fc52a19f..5cd5a954 100644 --- a/test/src/ManagedObjectTest.cc +++ b/test/src/ManagedObjectTest.cc @@ -66,7 +66,7 @@ class DestructWithGc : public ScriptClass { public: explicit DestructWithGc(const Local& thiz) : ScriptClass(thiz) {} - protected: +// protected: ~DestructWithGc() override { getScriptEngine()->gc(); } }; diff --git a/test/src/NativeTest.cc b/test/src/NativeTest.cc index 709382f3..6a088263 100644 --- a/test/src/NativeTest.cc +++ b/test/src/NativeTest.cc @@ -106,12 +106,13 @@ void testStatic(ScriptEngine* engine) { // prop: get version auto getVersion = engine->eval(TS().js("script.engine.test.TestClass.version") .lua("return script.engine.test.TestClass.version") + .py("script.engine.test.TestClass.version") .select()); ASSERT_TRUE(getVersion.isString()); EXPECT_EQ(getVersion.asString().toString(), version); // prop: set version - engine->eval("script.engine.test.TestClass['version'] = '1.0-beta' "); + engine->eval("script.engine.test.TestClass.version = '1.0-beta'"); EXPECT_EQ(std::string("1.0-beta"), version); // function: add @@ -171,8 +172,10 @@ TEST_F(NativeTest, All) { script::EngineScope engineScope(engine); engine->registerNativeClass(TestClassDefAll); - auto ret = engine->eval( - TS().js("script.engine.test.TestClass").lua("return script.engine.test.TestClass").select()); + auto ret = engine->eval(TS().js("script.engine.test.TestClass") + .lua("return script.engine.test.TestClass") + .py("script.engine.test.TestClass") + .select()); ASSERT_TRUE(ret.isObject()); testStatic(engine); @@ -189,8 +192,10 @@ TEST_F(NativeTest, Static) { engine->registerNativeClass(TestClassDefStatic); - auto ret = engine->eval( - TS().js("script.engine.test.TestClass").lua("return script.engine.test.TestClass").select()); + auto ret = engine->eval(TS().js("script.engine.test.TestClass") + .lua("return script.engine.test.TestClass") + .py("script.engine.test.TestClass") + .select()); ASSERT_TRUE(ret.isObject()); testStatic(engine); @@ -203,6 +208,7 @@ TEST_F(NativeTest, Instance) { auto ret = engine->eval(TS().js("script.engine.test.TestClass") .lua("return script.engine.test.TestClass") + .py("script.engine.test.TestClass") .select()); ASSERT_TRUE(ret.isObject()); @@ -240,8 +246,10 @@ TEST_F(NativeTest, NativeRegister) { script::EngineScope engineScope(engine); auto reg = NativeRegisterDef.getNativeRegister(); reg.registerNativeClass(engine); - auto ret = engine->eval( - TS().js("script.engine.test.TestClass").lua("return script.engine.test.TestClass").select()); + auto ret = engine->eval(TS().js("script.engine.test.TestClass") + .lua("return script.engine.test.TestClass") + .py("script.engine.test.TestClass") + .select()); ASSERT_TRUE(ret.isObject()); testStatic(engine); @@ -260,8 +268,10 @@ TEST_F(NativeTest, NativeRegister2) { script::EngineScope engineScope(engine); auto reg = NativeRegisterDef.getNativeRegister(); engine->registerNativeClass(reg); - auto ret = engine->eval( - TS().js("script.engine.test.TestClass").lua("return script.engine.test.TestClass").select()); + auto ret = engine->eval(TS().js("script.engine.test.TestClass") + .lua("return script.engine.test.TestClass") + .py("script.engine.test.TestClass") + .select()); ASSERT_TRUE(ret.isObject()); testStatic(engine); @@ -1300,7 +1310,7 @@ static ClassDefine wasmScriptClassDestroyScri .constructor() .build(); -TEST_F(NativeTest, WasmScriptClassDestoryTest) { +TEST_F(NativeTest, WasmScriptClassDestroyTest) { EngineScope scope(engine); engine->registerNativeClass(wasmScriptClassDestroyScriptClassDefine); diff --git a/test/src/ValueTest.cc b/test/src/ValueTest.cc index 82330f51..05fab2f5 100644 --- a/test/src/ValueTest.cc +++ b/test/src/ValueTest.cc @@ -45,9 +45,8 @@ return f; )"; -TEST_F(ValueTest, Object_NewObject) { - EngineScope engineScope(engine); - Local func = engine->eval(TS().js(u8R"( +constexpr auto kJsClassScript = + u8R"( function f(name, age) { this.name = name; this.age = age; @@ -61,10 +60,17 @@ f.prototype.greet = function() { f; -)") - .lua(kLuaClassScript) - .select()); +)"; + +const auto kPyClassScript = + "{'name':'my name', 'age': 11, 'greet': lambda self : 'Hello, I\\'m '+self['name']+' " + "'+str(self['age'])+' years old.'}"; +#ifndef SCRIPTX_LANG_PYTHON +TEST_F(ValueTest, Object_NewObject) { + EngineScope engineScope(engine); + Local func = + engine->eval(TS().js(kJsClassScript).lua(kLuaClassScript).py(kPyClassScript).select()); ASSERT_TRUE(func.isObject()); std::initializer_list> jennyList{ @@ -95,6 +101,7 @@ f; ASSERT_STREQ(greetRet.asString().toString().c_str(), "Hello, I'm Jenny 5 years old."); } } +#endif TEST_F(ValueTest, Object) { EngineScope engineScope(engine); @@ -180,7 +187,10 @@ TEST_F(ValueTest, String) { EXPECT_STREQ(string, str.describeUtf8().c_str()); EXPECT_EQ(strVal, str); - str = engine->eval(TS().js("'hello world'").lua("return 'hello world'").select()).asString(); + str = + engine + ->eval(TS().js("'hello world'").lua("return 'hello world'").py("'hello world'").select()) + .asString(); EXPECT_STREQ(string, str.toString().c_str()); } @@ -193,7 +203,11 @@ TEST_F(ValueTest, U8String) { auto str = String::newString(string); EXPECT_EQ(string, str.toU8string()); - str = engine->eval(TS().js(u8"'你好, 世界'").lua(u8"return '你好, 世界'").select()).asString(); + str = + engine + ->eval( + TS().js(u8"'你好, 世界'").lua(u8"return '你好, 世界'").py(u8"'你好, 世界'").select()) + .asString(); EXPECT_EQ(string, str.toU8string()); } @@ -210,6 +224,10 @@ TEST_F(ValueTest, InstanceOf) { auto f = engine->eval(kLuaClassScript); auto ins = engine->eval("return f()"); EXPECT_TRUE(ins.asObject().instanceOf(f)); +#elif defined(SCRIPTX_LANG_PYTHON) + auto f = engine->eval("{}"); + auto ins = engine->eval("dict()"); + EXPECT_TRUE(ins.asObject().instanceOf(f)); #else FAIL() << "add test impl here"; #endif @@ -310,6 +328,8 @@ TEST_F(ValueTest, FunctionReceiver) { } TEST_F(ValueTest, FunctionCall) { +#ifndef SCRIPTX_LANG_PYTHON + EngineScope engineScope(engine); engine->eval(TS().js(R"( function unitTestFuncCall(arg1, arg2) { @@ -328,6 +348,7 @@ function unitTestFuncCall(arg1, arg2) end end )") + .py(R"(lambda arg1,arg2:)") .select()); auto func = engine->get("unitTestFuncCall").asFunction(); @@ -339,6 +360,7 @@ end ret = func.call({}, 1, "x"); ASSERT_TRUE(ret.isString()); EXPECT_STREQ(ret.asString().toString().c_str(), "hello X"); +#endif } TEST_F(ValueTest, FunctionReturn) { @@ -413,14 +435,18 @@ TEST_F(ValueTest, FunctionHasThiz) { engine->set("func", func); auto hasThiz = - engine->eval(TS().js("var x = {func: func}; x.func()").lua("return func()").select()) + engine + ->eval( + TS().js("var x = {func: func}; x.func()").lua("return func()").py("func()").select()) .asBoolean() .value(); #ifdef SCRIPTX_LANG_JAVASCRIPT EXPECT_TRUE(hasThiz); -#elif SCRIPTX_LANG_Lua +#elif defined(SCRIPTX_LANG_LUA) EXPECT_FALSE(hasThiz); +#elif defined(SCRIPTX_LANG_PYTHON) + EXPECT_TRUE(hasThiz); #endif } @@ -429,9 +455,11 @@ TEST_F(ValueTest, EngineEvalReturnValue) { Local val; #if defined(SCRIPTX_LANG_JAVASCRIPT) - val = engine->eval(R"(3.14)"); + val = engine->eval("3.14"); #elif defined(SCRIPTX_LANG_LUA) - val = engine->eval(R"(return 3.14)"); + val = engine->eval("return 3.14"); +#elif defined(SCRIPTX_LANG_PYTHON) + val = engine->eval("3.14"); #else FAIL(); #endif @@ -631,6 +659,8 @@ TEST_F(ValueTest, Unsupported) { auto lua = lua_interop::currentEngineLua(); lua_newuserdata(lua, 4); auto strange = lua_interop::makeLocal(lua_gettop(lua)); +#elif defined(SCRIPTX_LANG_PYTHON) + auto strange = py_interop::asLocal(PyImport_AddModule("__main__")); #else FAIL() << "add test here"; auto strange = Local(); diff --git a/test/src/test.h b/test/src/test.h index 6d5b814d..b3d10101 100644 --- a/test/src/test.h +++ b/test/src/test.h @@ -46,6 +46,15 @@ struct TS { return *this; } + template + TS& py(T&& s) { + static_cast(s); +#ifdef SCRIPTX_LANG_PYTHON + script = script::String::newString(std::forward(s)); +#endif + return *this; + } + script::Local select() { if (script.isNull()) { throw std::runtime_error("add script for current language");