From 2e375946fa712efade5767ed4f2b7684f3036873 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 1 Feb 2023 20:43:03 +0100 Subject: [PATCH 01/42] python: Initialise bindings from Pythonix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Copies part of the tree from Pythonix at https://github.com/Mic92/pythonix/commit/fbc84900b5dcdde558c68d98ab746d76e7884ded into the ./python subdirectory. These Python bindings don't build yet with the current Nix version, which is why they're not built yet in this commit. Mic92 confirmed that he's okay with the license being changed to the Nix one: https://github.com/NixOS/nix/pull/7735#discussion_r1094242801 Co-Authored-By: Jörg Thalheim --- python/meson.build | 13 +++ python/src/eval.cc | 60 ++++++++++ python/src/internal/errors.hh | 8 ++ python/src/internal/eval.hh | 8 ++ python/src/internal/nix-to-python.hh | 13 +++ python/src/internal/ptr.hh | 13 +++ python/src/internal/python-to-nix.hh | 15 +++ python/src/meson.build | 12 ++ python/src/nix-to-python.cc | 93 ++++++++++++++++ python/src/python-module.cc | 53 +++++++++ python/src/python-to-nix.cc | 159 +++++++++++++++++++++++++++ python/tests.py | 20 ++++ 12 files changed, 467 insertions(+) create mode 100644 python/meson.build create mode 100644 python/src/eval.cc create mode 100644 python/src/internal/errors.hh create mode 100644 python/src/internal/eval.hh create mode 100644 python/src/internal/nix-to-python.hh create mode 100644 python/src/internal/ptr.hh create mode 100644 python/src/internal/python-to-nix.hh create mode 100644 python/src/meson.build create mode 100644 python/src/nix-to-python.cc create mode 100644 python/src/python-module.cc create mode 100644 python/src/python-to-nix.cc create mode 100644 python/tests.py diff --git a/python/meson.build b/python/meson.build new file mode 100644 index 00000000000..f7d86300c47 --- /dev/null +++ b/python/meson.build @@ -0,0 +1,13 @@ +project('python-nix', 'cpp', + version : '0.1.8', + license : 'LGPL-2.0', +) + +python_mod = import('python3') +python_dep = dependency('python3', required : true) +nix_expr_dep = dependency('nix-expr', required: true) + +python = python_mod.find_python() +test('python test', python, args : files('tests.py')) + +subdir('src') diff --git a/python/src/eval.cc b/python/src/eval.cc new file mode 100644 index 00000000000..b3e5af118ee --- /dev/null +++ b/python/src/eval.cc @@ -0,0 +1,60 @@ +#include "internal/eval.hh" +#include "internal/errors.hh" +#include "internal/nix-to-python.hh" +#include "internal/python-to-nix.hh" +#include + +#include +#include + +namespace pythonnix { + +const char *currentExceptionTypeName() { + int status; + auto res = abi::__cxa_demangle(abi::__cxa_current_exception_type()->name(), 0, + 0, &status); + return res ? res : "(null)"; +} + +static PyObject *_eval(const char *expression, PyObject *vars) { + nix::Strings storePath; + nix::EvalState state(storePath, nix::openStore()); + + nix::Env *env = nullptr; + auto staticEnv = pythonToNixEnv(state, vars, &env); + if (!staticEnv) { + return nullptr; + } + + auto e = state.parseExprFromString(expression, ".", *staticEnv); + nix::Value v; + e->eval(state, *env, v); + + state.forceValueDeep(v); + + nix::PathSet context; + return nixToPythonObject(state, v, context); +} + +PyObject *eval(PyObject *self, PyObject *args, PyObject *keywds) { + const char *expression = nullptr; + PyObject *vars = nullptr; + + const char *kwlist[] = {"expression", "vars", nullptr}; + + if (!PyArg_ParseTupleAndKeywords(args, keywds, "s|O!", + const_cast(kwlist), &expression, + &PyDict_Type, &vars)) { + return nullptr; + } + + try { + return _eval(expression, vars); + } catch (nix::Error &e) { + return PyErr_Format(NixError, "%s", e.what()); + } catch (...) { + return PyErr_Format(NixError, "unexpected C++ exception: '%s'", + currentExceptionTypeName()); + } +} +} // namespace pythonnix diff --git a/python/src/internal/errors.hh b/python/src/internal/errors.hh new file mode 100644 index 00000000000..86d0cf577bc --- /dev/null +++ b/python/src/internal/errors.hh @@ -0,0 +1,8 @@ +#pragma once + +#include + +namespace pythonnix { + +extern PyObject *NixError; +} diff --git a/python/src/internal/eval.hh b/python/src/internal/eval.hh new file mode 100644 index 00000000000..dc6fe1f7580 --- /dev/null +++ b/python/src/internal/eval.hh @@ -0,0 +1,8 @@ +#pragma once + +#include + +namespace pythonnix { + +PyObject *eval(PyObject *self, PyObject *args, PyObject *kwdict); +} diff --git a/python/src/internal/nix-to-python.hh b/python/src/internal/nix-to-python.hh new file mode 100644 index 00000000000..27849fd0329 --- /dev/null +++ b/python/src/internal/nix-to-python.hh @@ -0,0 +1,13 @@ +#pragma once + +#include + +#include + +#include + +namespace pythonnix { + +PyObject *nixToPythonObject(nix::EvalState &state, nix::Value &v, + nix::PathSet &context); +} // namespace pythonnix diff --git a/python/src/internal/ptr.hh b/python/src/internal/ptr.hh new file mode 100644 index 00000000000..16558772848 --- /dev/null +++ b/python/src/internal/ptr.hh @@ -0,0 +1,13 @@ +#pragma once + +#include +#include + +namespace pythonnix { + +struct PyObjectDeleter { + void operator()(PyObject *const obj) { Py_DECREF(obj); } +}; + +typedef std::unique_ptr PyObjPtr; +} // namespace pythonnix diff --git a/python/src/internal/python-to-nix.hh b/python/src/internal/python-to-nix.hh new file mode 100644 index 00000000000..c0ddaf012e1 --- /dev/null +++ b/python/src/internal/python-to-nix.hh @@ -0,0 +1,15 @@ +#pragma once + +#include +#include + +#include +#include + +namespace pythonnix { + +nix::Value *pythonToNixValue(nix::EvalState &state, PyObject *obj); + +std::optional pythonToNixEnv(nix::EvalState &state, + PyObject *vars, nix::Env **env); +} // namespace pythonnix diff --git a/python/src/meson.build b/python/src/meson.build new file mode 100644 index 00000000000..a50f866d84b --- /dev/null +++ b/python/src/meson.build @@ -0,0 +1,12 @@ +src = [ + 'nix-to-python.cc', + 'python-to-nix.cc', + 'eval.cc', + 'python-module.cc', +] + +python_mod.extension_module('nix', src, + dependencies : [python_dep, nix_expr_dep], + install: true, + install_dir: python_mod.sysconfig_path('platlib'), + cpp_args: ['-std=c++17', '-fvisibility=hidden']) diff --git a/python/src/nix-to-python.cc b/python/src/nix-to-python.cc new file mode 100644 index 00000000000..399f8d6f2ce --- /dev/null +++ b/python/src/nix-to-python.cc @@ -0,0 +1,93 @@ +#include + +#include "internal/errors.hh" +#include "internal/nix-to-python.hh" +#include "internal/ptr.hh" + +namespace pythonnix { + +PyObject *nixToPythonObject(nix::EvalState &state, nix::Value &v, + nix::PathSet &context) { + switch (v.type()) { + case nix::nInt: + return PyLong_FromLong(v.integer); + + case nix::nBool: + if (v.boolean) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } + case nix::nString: + copyContext(v, context); + return PyUnicode_FromString(v.string.s); + + case nix::nPath: + return PyUnicode_FromString(state.copyPathToStore(context, v.path).c_str()); + + case nix::nNull: + Py_RETURN_NONE; + + case nix::nAttrs: { + auto i = v.attrs->find(state.sOutPath); + if (i == v.attrs->end()) { + PyObjPtr dict(PyDict_New()); + if (!dict) { + return (PyObject *)nullptr; + } + + nix::StringSet names; + + for (auto &j : *v.attrs) { + names.insert(j.name); + } + for (auto &j : names) { + nix::Attr &a(*v.attrs->find(state.symbols.create(j))); + + auto value = nixToPythonObject(state, *a.value, context); + if (!value) { + return nullptr; + } + PyDict_SetItemString(dict.get(), j.c_str(), value); + } + return dict.release(); + } else { + return nixToPythonObject(state, *i->value, context); + } + } + + case nix::nList: { + PyObjPtr list(PyList_New(v.listSize())); + if (!list) { + return (PyObject *)nullptr; + } + + for (unsigned int n = 0; n < v.listSize(); ++n) { + auto value = nixToPythonObject(state, *v.listElems()[n], context); + if (!value) { + return nullptr; + } + PyList_SET_ITEM(list.get(), n, value); + } + return list.release(); + } + + case nix::nExternal: + return PyUnicode_FromString(""); + + case nix::nThunk: + return PyUnicode_FromString(""); + + case nix::nFunction: + return PyUnicode_FromString(""); + + case nix::nFloat: + return PyFloat_FromDouble(v.fpoint); + + default: + PyErr_Format(NixError, "cannot convert nix type '%s' to a python object", + showType(v).c_str()); + return nullptr; + } +} +} // namespace pythonnix diff --git a/python/src/python-module.cc b/python/src/python-module.cc new file mode 100644 index 00000000000..4dfd5aa93a8 --- /dev/null +++ b/python/src/python-module.cc @@ -0,0 +1,53 @@ +#include + +#include "internal/eval.hh" +#include "internal/ptr.hh" + +#include + +#include + +namespace pythonnix { + +#define _public_ __attribute__((visibility("default"))) + +PyObject *NixError = nullptr; + +static PyMethodDef NixMethods[] = {{"eval", (PyCFunction)eval, + METH_VARARGS | METH_KEYWORDS, + "Eval nix expression"}, + {NULL, NULL, 0, NULL}}; + +static struct PyModuleDef nixmodule = { + PyModuleDef_HEAD_INIT, "nix", "Nix expression bindings", + -1, /* size of per-interpreter state of the module, + or -1 if the module keeps state in global variables. */ + NixMethods}; + +extern "C" _public_ PyObject *PyInit_nix(void) { + nix::initGC(); + + PyObjPtr m(PyModule_Create(&nixmodule)); + + if (!m) { + return nullptr; + } + + NixError = PyErr_NewExceptionWithDoc( + "nix.NixError", /* char *name */ + "Base exception class for the nix module.", /* char *doc */ + NULL, /* PyObject *base */ + NULL /* PyObject *dict */ + ); + + if (!NixError) { + return nullptr; + } + + if (PyModule_AddObject(m.get(), "NixError", NixError) == -1) { + return nullptr; + } + + return m.release(); +} +} // namespace pythonnix diff --git a/python/src/python-to-nix.cc b/python/src/python-to-nix.cc new file mode 100644 index 00000000000..f79134dfa4a --- /dev/null +++ b/python/src/python-to-nix.cc @@ -0,0 +1,159 @@ +#include + +#include "internal/errors.hh" +#include "internal/ptr.hh" +#include "internal/python-to-nix.hh" + +#include + +namespace pythonnix { + +static const char *checkNullByte(const char *str, const Py_ssize_t size) { + for (Py_ssize_t i = 0; i < size; i++) { + if (str[0] == '\0') { + PyErr_Format(NixError, "invalid character: nix strings are not allowed " + "to contain null bytes"); + return nullptr; + } + } + return str; +} + +static const char *checkAttrKey(PyObject *obj) { + Py_ssize_t size = 0; + + if (!PyUnicode_Check(obj)) { + PyObjPtr typeName(PyObject_Str(PyObject_Type(obj))); + if (!typeName) { + return nullptr; + } + auto utf8 = PyUnicode_AsUTF8AndSize(typeName.get(), &size); + if (!utf8) { + return nullptr; + } + PyErr_Format(NixError, "key of nix attrsets must be strings, got type: %s", + utf8); + return nullptr; + } + + auto utf8 = PyUnicode_AsUTF8AndSize(obj, &size); + if (!utf8) { + return nullptr; + } + + return checkNullByte(utf8, size); +} + +static std::optional dictToAttrSet(PyObject *obj, + nix::EvalState &state) { + PyObject *key = nullptr, *val = nullptr; + Py_ssize_t pos = 0; + + nix::ValueMap attrs; + while (PyDict_Next(obj, &pos, &key, &val)) { + auto name = checkAttrKey(key); + if (!name) { + return {}; + } + + auto attrVal = pythonToNixValue(state, val); + if (!attrVal) { + return {}; + } + attrs[state.symbols.create(name)] = attrVal; + } + + return attrs; +} + +nix::Value *pythonToNixValue(nix::EvalState &state, PyObject *obj) { + auto v = state.allocValue(); + + if (obj == Py_True && obj == Py_False) { + nix::mkBool(*v, obj == Py_True); + } else if (obj == Py_None) { + nix::mkNull(*v); + } else if (PyBytes_Check(obj)) { + auto str = checkNullByte(PyBytes_AS_STRING(obj), PyBytes_GET_SIZE(obj)); + if (!str) { + return nullptr; + } + + nix::mkString(*v, str); + } else if (PyUnicode_Check(obj)) { + Py_ssize_t size; + const char *utf8 = PyUnicode_AsUTF8AndSize(obj, &size); + auto str = checkNullByte(utf8, size); + if (!str) { + return nullptr; + } + + nix::mkString(*v, utf8); + } else if (PyFloat_Check(obj)) { + nix::mkFloat(*v, PyFloat_AS_DOUBLE(obj)); + } else if (PyLong_Check(obj)) { + nix::mkInt(*v, PyLong_AsLong(obj)); + } else if (PyList_Check(obj)) { + state.mkList(*v, PyList_GET_SIZE(obj)); + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(obj); i++) { + auto val = pythonToNixValue(state, PyList_GET_ITEM(obj, i)); + if (!val) { + return nullptr; + } + v->listElems()[i] = val; + } + } else if (PyTuple_Check(obj)) { + state.mkList(*v, PyTuple_GET_SIZE(obj)); + for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(obj); i++) { + auto val = pythonToNixValue(state, PyTuple_GET_ITEM(obj, i)); + if (!val) { + return nullptr; + } + v->listElems()[i] = val; + } + } else if (PyDict_Check(obj)) { + auto attrs = dictToAttrSet(obj, state); + if (!attrs) { + return nullptr; + } + state.mkAttrs(*v, attrs->size()); + for (auto &attr : *attrs) { + v->attrs->push_back(nix::Attr(attr.first, attr.second)); + } + v->attrs->sort(); + } + return v; +} + +std::optional pythonToNixEnv(nix::EvalState &state, + PyObject *vars, nix::Env **env) { + Py_ssize_t pos = 0; + PyObject *key = nullptr, *val = nullptr; + + *env = &state.allocEnv(vars ? PyDict_Size(vars) : 0); + (*env)->up = &state.baseEnv; + + nix::StaticEnv staticEnv(false, &state.staticBaseEnv); + + if (!vars) { + return staticEnv; + } + + auto displ = 0; + while (PyDict_Next(vars, &pos, &key, &val)) { + auto name = checkAttrKey(key); + if (!name) { + return {}; + } + + auto attrVal = pythonToNixValue(state, val); + if (!attrVal) { + return {}; + } + staticEnv.vars[state.symbols.create(name)] = displ; + (*env)->values[displ++] = attrVal; + } + + return staticEnv; +} +} // namespace pythonnix diff --git a/python/tests.py b/python/tests.py new file mode 100644 index 00000000000..9d909e98a73 --- /dev/null +++ b/python/tests.py @@ -0,0 +1,20 @@ +import nix +import unittest + + +class TestPythonNix(unittest.TestCase): + def test_dict(self): + val = dict(a=1) + self.assertEqual(nix.eval("a", vars=dict(a=val)), val) + + def test_string(self): + self.assertEqual(nix.eval("a", vars=dict(a="foo")), "foo") + + def test_bool(self): + self.assertEqual(nix.eval("a", vars=dict(a=True)), True) + + def test_none(self): + self.assertEqual(nix.eval("a", vars=dict(a=None)), None) + +if __name__ == '__main__': + unittest.main() From ed8c80b5667be752fa866cdf93ae4cac207fbff8 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 17 Feb 2023 19:41:03 +0100 Subject: [PATCH 02/42] python: Fixes for Nix changes The python bindings initialised from Pythonix haven't been updated in some Nix versions and would not compile anymore. This commit still doesn't include a working python bindings build, but these are the Nix-update-caused changes that will be necessary for that --- python/src/eval.cc | 3 ++- python/src/nix-to-python.cc | 18 +++++++----------- python/src/python-module.cc | 3 +++ python/src/python-to-nix.cc | 29 +++++++++++++++-------------- 4 files changed, 27 insertions(+), 26 deletions(-) diff --git a/python/src/eval.cc b/python/src/eval.cc index b3e5af118ee..ec15d0c0eb3 100644 --- a/python/src/eval.cc +++ b/python/src/eval.cc @@ -25,8 +25,9 @@ static PyObject *_eval(const char *expression, PyObject *vars) { if (!staticEnv) { return nullptr; } + auto staticEnvPointer = std::make_shared(*staticEnv); - auto e = state.parseExprFromString(expression, ".", *staticEnv); + auto e = state.parseExprFromString(expression, ".", staticEnvPointer); nix::Value v; e->eval(state, *env, v); diff --git a/python/src/nix-to-python.cc b/python/src/nix-to-python.cc index 399f8d6f2ce..63793f56ac9 100644 --- a/python/src/nix-to-python.cc +++ b/python/src/nix-to-python.cc @@ -22,8 +22,10 @@ PyObject *nixToPythonObject(nix::EvalState &state, nix::Value &v, copyContext(v, context); return PyUnicode_FromString(v.string.s); - case nix::nPath: - return PyUnicode_FromString(state.copyPathToStore(context, v.path).c_str()); + case nix::nPath: { + auto p = state.copyPathToStore(context, v.path).to_string(); + return PyUnicode_FromStringAndSize(p.data(), p.length()); + } case nix::nNull: Py_RETURN_NONE; @@ -36,19 +38,13 @@ PyObject *nixToPythonObject(nix::EvalState &state, nix::Value &v, return (PyObject *)nullptr; } - nix::StringSet names; - for (auto &j : *v.attrs) { - names.insert(j.name); - } - for (auto &j : names) { - nix::Attr &a(*v.attrs->find(state.symbols.create(j))); - - auto value = nixToPythonObject(state, *a.value, context); + const std::string & name = state.symbols[j.name]; + auto value = nixToPythonObject(state, *j.value, context); if (!value) { return nullptr; } - PyDict_SetItemString(dict.get(), j.c_str(), value); + PyDict_SetItemString(dict.get(), name.c_str(), value); } return dict.release(); } else { diff --git a/python/src/python-module.cc b/python/src/python-module.cc index 4dfd5aa93a8..4ff8ad01f89 100644 --- a/python/src/python-module.cc +++ b/python/src/python-module.cc @@ -6,6 +6,8 @@ #include #include +#include +#include namespace pythonnix { @@ -25,6 +27,7 @@ static struct PyModuleDef nixmodule = { NixMethods}; extern "C" _public_ PyObject *PyInit_nix(void) { + nix::initNix(); nix::initGC(); PyObjPtr m(PyModule_Create(&nixmodule)); diff --git a/python/src/python-to-nix.cc b/python/src/python-to-nix.cc index f79134dfa4a..f6326eef488 100644 --- a/python/src/python-to-nix.cc +++ b/python/src/python-to-nix.cc @@ -70,16 +70,16 @@ nix::Value *pythonToNixValue(nix::EvalState &state, PyObject *obj) { auto v = state.allocValue(); if (obj == Py_True && obj == Py_False) { - nix::mkBool(*v, obj == Py_True); + v->mkBool(obj == Py_True); } else if (obj == Py_None) { - nix::mkNull(*v); + v->mkNull(); } else if (PyBytes_Check(obj)) { auto str = checkNullByte(PyBytes_AS_STRING(obj), PyBytes_GET_SIZE(obj)); if (!str) { return nullptr; } - nix::mkString(*v, str); + v->mkString(str); } else if (PyUnicode_Check(obj)) { Py_ssize_t size; const char *utf8 = PyUnicode_AsUTF8AndSize(obj, &size); @@ -88,13 +88,13 @@ nix::Value *pythonToNixValue(nix::EvalState &state, PyObject *obj) { return nullptr; } - nix::mkString(*v, utf8); + v->mkString(utf8); } else if (PyFloat_Check(obj)) { - nix::mkFloat(*v, PyFloat_AS_DOUBLE(obj)); + v->mkFloat(PyFloat_AS_DOUBLE(obj)); } else if (PyLong_Check(obj)) { - nix::mkInt(*v, PyLong_AsLong(obj)); + v->mkInt(PyLong_AsLong(obj)); } else if (PyList_Check(obj)) { - state.mkList(*v, PyList_GET_SIZE(obj)); + v->mkList(PyList_GET_SIZE(obj)); for (Py_ssize_t i = 0; i < PyList_GET_SIZE(obj); i++) { auto val = pythonToNixValue(state, PyList_GET_ITEM(obj, i)); if (!val) { @@ -103,7 +103,7 @@ nix::Value *pythonToNixValue(nix::EvalState &state, PyObject *obj) { v->listElems()[i] = val; } } else if (PyTuple_Check(obj)) { - state.mkList(*v, PyTuple_GET_SIZE(obj)); + v->mkList(PyTuple_GET_SIZE(obj)); for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(obj); i++) { auto val = pythonToNixValue(state, PyTuple_GET_ITEM(obj, i)); if (!val) { @@ -116,11 +116,12 @@ nix::Value *pythonToNixValue(nix::EvalState &state, PyObject *obj) { if (!attrs) { return nullptr; } - state.mkAttrs(*v, attrs->size()); - for (auto &attr : *attrs) { - v->attrs->push_back(nix::Attr(attr.first, attr.second)); + auto attrsValue = attrs.value(); + auto bindings = state.buildBindings(attrsValue.size()); + for (auto &attr : attrsValue) { + bindings.insert(attr.first, attr.second); } - v->attrs->sort(); + v->mkAttrs(bindings); } return v; } @@ -133,7 +134,7 @@ std::optional pythonToNixEnv(nix::EvalState &state, *env = &state.allocEnv(vars ? PyDict_Size(vars) : 0); (*env)->up = &state.baseEnv; - nix::StaticEnv staticEnv(false, &state.staticBaseEnv); + nix::StaticEnv staticEnv(false, state.staticBaseEnv.get()); if (!vars) { return staticEnv; @@ -150,7 +151,7 @@ std::optional pythonToNixEnv(nix::EvalState &state, if (!attrVal) { return {}; } - staticEnv.vars[state.symbols.create(name)] = displ; + staticEnv.vars.emplace_back(state.symbols.create(name), displ); (*env)->values[displ++] = attrVal; } From 5dc8bcbb1ffe3c234b40f5df39b09deef22ec7e3 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 17 Feb 2023 19:44:46 +0100 Subject: [PATCH 03/42] python: Integrate incremental and CI build Makes the python bindings build, both incrementally and for CI. Documentation is not yet included --- flake.nix | 9 +++++ python/.gitignore | 2 ++ python/Makefile | 49 ++++++++++++++++++++++++++++ python/default.nix | 45 +++++++++++++++++++++++++ python/meson.build | 14 ++++++-- python/src/internal/nix-to-python.hh | 2 +- python/src/internal/python-to-nix.hh | 2 +- python/src/meson.build | 12 +++++-- python/src/python-module.cc | 2 +- python/test.sh | 7 ++++ 10 files changed, 135 insertions(+), 9 deletions(-) create mode 100644 python/.gitignore create mode 100644 python/Makefile create mode 100644 python/default.nix create mode 100644 python/test.sh diff --git a/flake.nix b/flake.nix index a4ee80b32c8..9af342b1c88 100644 --- a/flake.nix +++ b/flake.nix @@ -439,6 +439,11 @@ postUnpack = "sourceRoot=$sourceRoot/perl"; }); + passthru.python-bindings = final.callPackage ./python { + inherit self system; + python = final.python3; + }; + meta.platforms = lib.platforms.unix; }); @@ -500,6 +505,8 @@ # Perl bindings for various platforms. perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nix.perl-bindings); + pythonBindings = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix.python-bindings); + # Binary tarball for various platforms, containing a Nix store # with the closure of 'nix' package, and the second half of # the installation script. @@ -645,6 +652,7 @@ checks = forAllSystems (system: { binaryTarball = self.hydraJobs.binaryTarball.${system}; perlBindings = self.hydraJobs.perlBindings.${system}; + pythonBindings = self.hydraJobs.pythonBindings.${system}; installTests = self.hydraJobs.installTests.${system}; nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system}; } // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) { @@ -727,6 +735,7 @@ (forAllCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv)) // { default = self.devShells.${system}.native-stdenvPackages; + python = self.packages.${system}.nix.python-bindings.shell; } ); }; diff --git a/python/.gitignore b/python/.gitignore new file mode 100644 index 00000000000..7194ea7277e --- /dev/null +++ b/python/.gitignore @@ -0,0 +1,2 @@ +.cache +build diff --git a/python/Makefile b/python/Makefile new file mode 100644 index 00000000000..75c8f61b2d8 --- /dev/null +++ b/python/Makefile @@ -0,0 +1,49 @@ +# This Makefile is only used for development of the Python bindings, it is not +# used in the Nix build. +# The reason this exists is to make it easier to develop the Python bindings in +# tandem with the main Nix. +# The default `make` (defaults to `make build`) calls the main Nix projects +# `make install` before calling the Python bindings' `meson compile`, therefore +# ensuring that the needed Nix dynamic libraries are up-to-date + +builddir=build + +.PHONY: build +build: nix-install setup-done + meson compile -C $(builddir) + +.PHONY: test +test: nix-install setup-done + meson test -C $(builddir) -v + +.PHONY: clean +clean: + rm -rf $(builddir) + +# We include the main Nix projects Makefile.config file to know the $(libdir) +# variable, which is where Nix is installed in, which we can then use to setup +# the meson build +include ../Makefile.config + +# We need the file to exist though +../Makefile.config: + @# Throw a good error message in case ./configure hasn't been run yet + @[[ -e ../config.status ]] || ( echo "The main Nix project needs to be configured first, see https://nixos.org/manual/nix/stable/contributing/hacking.html" && exit 1 ) + @# If ./configure is done, we can create the file ourselves + $(MAKE) -C .. Makefile.config + +.PHONY: setup +setup: nix-install + @# Make meson be able to find the locally-installed Nix + PKG_CONFIG_PATH=$(libdir)/pkgconfig:$$PKG_CONFIG_PATH meson setup $(builddir) + +.PHONY: setup-done +setup-done: + @# A better error message in case the build directory doesn't exist yet + @[[ -e $(builddir) ]] || ( echo "Run 'make setup' once to configure the project build directory" && exit 1 ) + +.PHONY: nix-install +nix-install: + @# The python bindings don't technically need an _entire_ Nix installation, + @# but it seems non-trivial to pick out only exactly the files it actually needs + $(MAKE) -C .. install diff --git a/python/default.nix b/python/default.nix new file mode 100644 index 00000000000..f5a3132d5e7 --- /dev/null +++ b/python/default.nix @@ -0,0 +1,45 @@ +{ self, system, lib, python, ninja, meson, nix, mkShell }: +python.pkgs.buildPythonPackage { + name = "nix"; + format = "other"; + + src = self; + + strictDeps = true; + + nativeBuildInputs = lib.optionals (nix != null) nix.nativeBuildInputs ++ [ + ninja + (meson.override { python3 = python; }) + nix + ]; + + buildInputs = lib.optionals (nix != null) nix.buildInputs ++ [ + nix + ]; + + # We need to be able to generate tests/common.sh, which requires running + # `make`, which requires having run `autoreconf` and `./configure`. + # So we propagate `autoreconfHook` from nix.nativeBuildInputs for that to + # work, but after that we also need to cd into the python directory and run the + # meson configure phase for the python bindings. + # Can't use `postConfigure` for this because that would create a loop since + # `mesonConfigurePhase` calls `postConfigure` itself. + # A small problem with this is that `{pre,post}Configure` get run twice + dontUseMesonConfigure = true; + preBuild = '' + cd python + mesonConfigurePhase + ''; + + mesonBuildType = "release"; + + doInstallCheck = true; + installCheckPhase = "meson test -v"; + + passthru.shell = mkShell { + inputsFrom = [ + self.devShells.${system}.default + (nix.python-bindings.override { nix = null; }) + ]; + }; +} diff --git a/python/meson.build b/python/meson.build index f7d86300c47..97ba97e7309 100644 --- a/python/meson.build +++ b/python/meson.build @@ -6,8 +6,16 @@ project('python-nix', 'cpp', python_mod = import('python3') python_dep = dependency('python3', required : true) nix_expr_dep = dependency('nix-expr', required: true) - -python = python_mod.find_python() -test('python test', python, args : files('tests.py')) +nix_main_dep = dependency('nix-main', required: true) subdir('src') + +fs = import('fs') + +nix_root = fs.parent(meson.project_source_root()) +run_command('make', '-C', nix_root, 'tests/common/vars-and-functions.sh', check: true) + +env = environment() +env.prepend('PYTHONPATH', fs.parent(pythonix.full_path())) +bash = find_program('bash') +test('python test', bash, args : files('test.sh'), env : env) diff --git a/python/src/internal/nix-to-python.hh b/python/src/internal/nix-to-python.hh index 27849fd0329..a6266fa90f3 100644 --- a/python/src/internal/nix-to-python.hh +++ b/python/src/internal/nix-to-python.hh @@ -2,7 +2,7 @@ #include -#include +#include #include diff --git a/python/src/internal/python-to-nix.hh b/python/src/internal/python-to-nix.hh index c0ddaf012e1..db14b7f7533 100644 --- a/python/src/internal/python-to-nix.hh +++ b/python/src/internal/python-to-nix.hh @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include #include diff --git a/python/src/meson.build b/python/src/meson.build index a50f866d84b..937d755913b 100644 --- a/python/src/meson.build +++ b/python/src/meson.build @@ -5,8 +5,14 @@ src = [ 'python-module.cc', ] -python_mod.extension_module('nix', src, - dependencies : [python_dep, nix_expr_dep], +pythonix = python_mod.extension_module('nix', src, + dependencies : [python_dep, nix_expr_dep, nix_main_dep], install: true, install_dir: python_mod.sysconfig_path('platlib'), - cpp_args: ['-std=c++17', '-fvisibility=hidden']) + cpp_args: [ + '-std=c++17', + # -Wnon-virtual-dtor is unnecessarily turned on by Meson + # This is fixed in Meson 1.0.0 with https://github.com/mesonbuild/meson/pull/10339 + '-Wno-non-virtual-dtor', + '-fvisibility=hidden' + ]) diff --git a/python/src/python-module.cc b/python/src/python-module.cc index 4ff8ad01f89..32940dff9af 100644 --- a/python/src/python-module.cc +++ b/python/src/python-module.cc @@ -3,7 +3,7 @@ #include "internal/eval.hh" #include "internal/ptr.hh" -#include +#include #include #include diff --git a/python/test.sh b/python/test.sh new file mode 100644 index 00000000000..ca13ce5f6ea --- /dev/null +++ b/python/test.sh @@ -0,0 +1,7 @@ +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +cd "$SCRIPT_DIR"/../tests + +source init.sh + +python "$SCRIPT_DIR"/tests.py From de148e41b50e776dae242c9a00a96abd01af7a7c Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 17 Feb 2023 19:33:45 +0100 Subject: [PATCH 04/42] python: Fix for IFD --- python/src/python-module.cc | 5 +++++ python/tests.py | 11 +++++++++++ 2 files changed, 16 insertions(+) diff --git a/python/src/python-module.cc b/python/src/python-module.cc index 32940dff9af..25806929bdb 100644 --- a/python/src/python-module.cc +++ b/python/src/python-module.cc @@ -27,6 +27,11 @@ static struct PyModuleDef nixmodule = { NixMethods}; extern "C" _public_ PyObject *PyInit_nix(void) { + // By default, Nix sets the build-hook to be "$(readlink /proc/self/exe) __build-remote", expecting the current binary to be Nix itself. + // But when we call the Nix library from Python this isn't the case, the current binary is Python then + // So we need to change this default, pointing it to the Nix binary instead + nix::settings.buildHook = nix::settings.nixBinDir + "/nix __build-remote"; + // And by setting buildHook before calling initNix, we can override the defaults without overriding the user-provided options from the config files nix::initNix(); nix::initGC(); diff --git a/python/tests.py b/python/tests.py index 9d909e98a73..d2c4b1301ab 100644 --- a/python/tests.py +++ b/python/tests.py @@ -16,5 +16,16 @@ def test_bool(self): def test_none(self): self.assertEqual(nix.eval("a", vars=dict(a=None)), None) + def test_ifd(self): + expression = """ + builtins.readFile (derivation { + name = "test"; + args = [ "-c" "printf \\"%s\\" test > $out" ]; + builder = "/bin/sh"; + system = builtins.currentSystem; + }) + """ + self.assertEqual(nix.eval(expression, vars=dict()), "test") + if __name__ == '__main__': unittest.main() From 827f6342cc0996f2a78c44255e33165ed3b66c5b Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 17 Feb 2023 20:25:13 +0100 Subject: [PATCH 05/42] python: Format using clang-format Using the configuration file from #6721 for less conflicts --- python/src/eval.cc | 80 ++++----- python/src/internal/errors.hh | 2 +- python/src/internal/eval.hh | 2 +- python/src/internal/nix-to-python.hh | 3 +- python/src/internal/ptr.hh | 8 +- python/src/internal/python-to-nix.hh | 5 +- python/src/nix-to-python.cc | 127 +++++++------- python/src/python-module.cc | 72 ++++---- python/src/python-to-nix.cc | 254 ++++++++++++++------------- 9 files changed, 279 insertions(+), 274 deletions(-) diff --git a/python/src/eval.cc b/python/src/eval.cc index ec15d0c0eb3..e95f7c88b96 100644 --- a/python/src/eval.cc +++ b/python/src/eval.cc @@ -9,53 +9,53 @@ namespace pythonnix { -const char *currentExceptionTypeName() { - int status; - auto res = abi::__cxa_demangle(abi::__cxa_current_exception_type()->name(), 0, - 0, &status); - return res ? res : "(null)"; +const char * currentExceptionTypeName() +{ + int status; + auto res = abi::__cxa_demangle(abi::__cxa_current_exception_type()->name(), 0, 0, &status); + return res ? res : "(null)"; } -static PyObject *_eval(const char *expression, PyObject *vars) { - nix::Strings storePath; - nix::EvalState state(storePath, nix::openStore()); +static PyObject * _eval(const char * expression, PyObject * vars) +{ + nix::Strings storePath; + nix::EvalState state(storePath, nix::openStore()); - nix::Env *env = nullptr; - auto staticEnv = pythonToNixEnv(state, vars, &env); - if (!staticEnv) { - return nullptr; - } - auto staticEnvPointer = std::make_shared(*staticEnv); + nix::Env * env = nullptr; + auto staticEnv = pythonToNixEnv(state, vars, &env); + if (!staticEnv) { + return nullptr; + } + auto staticEnvPointer = std::make_shared(*staticEnv); - auto e = state.parseExprFromString(expression, ".", staticEnvPointer); - nix::Value v; - e->eval(state, *env, v); + auto e = state.parseExprFromString(expression, ".", staticEnvPointer); + nix::Value v; + e->eval(state, *env, v); - state.forceValueDeep(v); + state.forceValueDeep(v); - nix::PathSet context; - return nixToPythonObject(state, v, context); + nix::PathSet context; + return nixToPythonObject(state, v, context); } -PyObject *eval(PyObject *self, PyObject *args, PyObject *keywds) { - const char *expression = nullptr; - PyObject *vars = nullptr; - - const char *kwlist[] = {"expression", "vars", nullptr}; - - if (!PyArg_ParseTupleAndKeywords(args, keywds, "s|O!", - const_cast(kwlist), &expression, - &PyDict_Type, &vars)) { - return nullptr; - } - - try { - return _eval(expression, vars); - } catch (nix::Error &e) { - return PyErr_Format(NixError, "%s", e.what()); - } catch (...) { - return PyErr_Format(NixError, "unexpected C++ exception: '%s'", - currentExceptionTypeName()); - } +PyObject * eval(PyObject * self, PyObject * args, PyObject * keywds) +{ + const char * expression = nullptr; + PyObject * vars = nullptr; + + const char * kwlist[] = {"expression", "vars", nullptr}; + + if (!PyArg_ParseTupleAndKeywords( + args, keywds, "s|O!", const_cast(kwlist), &expression, &PyDict_Type, &vars)) { + return nullptr; + } + + try { + return _eval(expression, vars); + } catch (nix::Error & e) { + return PyErr_Format(NixError, "%s", e.what()); + } catch (...) { + return PyErr_Format(NixError, "unexpected C++ exception: '%s'", currentExceptionTypeName()); + } } } // namespace pythonnix diff --git a/python/src/internal/errors.hh b/python/src/internal/errors.hh index 86d0cf577bc..a4f6bd81e0b 100644 --- a/python/src/internal/errors.hh +++ b/python/src/internal/errors.hh @@ -4,5 +4,5 @@ namespace pythonnix { -extern PyObject *NixError; +extern PyObject * NixError; } diff --git a/python/src/internal/eval.hh b/python/src/internal/eval.hh index dc6fe1f7580..878b82ff643 100644 --- a/python/src/internal/eval.hh +++ b/python/src/internal/eval.hh @@ -4,5 +4,5 @@ namespace pythonnix { -PyObject *eval(PyObject *self, PyObject *args, PyObject *kwdict); +PyObject * eval(PyObject * self, PyObject * args, PyObject * kwdict); } diff --git a/python/src/internal/nix-to-python.hh b/python/src/internal/nix-to-python.hh index a6266fa90f3..bfcc8bf22ba 100644 --- a/python/src/internal/nix-to-python.hh +++ b/python/src/internal/nix-to-python.hh @@ -8,6 +8,5 @@ namespace pythonnix { -PyObject *nixToPythonObject(nix::EvalState &state, nix::Value &v, - nix::PathSet &context); +PyObject * nixToPythonObject(nix::EvalState & state, nix::Value & v, nix::PathSet & context); } // namespace pythonnix diff --git a/python/src/internal/ptr.hh b/python/src/internal/ptr.hh index 16558772848..5ca073de283 100644 --- a/python/src/internal/ptr.hh +++ b/python/src/internal/ptr.hh @@ -5,8 +5,12 @@ namespace pythonnix { -struct PyObjectDeleter { - void operator()(PyObject *const obj) { Py_DECREF(obj); } +struct PyObjectDeleter +{ + void operator()(PyObject * const obj) + { + Py_DECREF(obj); + } }; typedef std::unique_ptr PyObjPtr; diff --git a/python/src/internal/python-to-nix.hh b/python/src/internal/python-to-nix.hh index db14b7f7533..3137336e964 100644 --- a/python/src/internal/python-to-nix.hh +++ b/python/src/internal/python-to-nix.hh @@ -8,8 +8,7 @@ namespace pythonnix { -nix::Value *pythonToNixValue(nix::EvalState &state, PyObject *obj); +nix::Value * pythonToNixValue(nix::EvalState & state, PyObject * obj); -std::optional pythonToNixEnv(nix::EvalState &state, - PyObject *vars, nix::Env **env); +std::optional pythonToNixEnv(nix::EvalState & state, PyObject * vars, nix::Env ** env); } // namespace pythonnix diff --git a/python/src/nix-to-python.cc b/python/src/nix-to-python.cc index 63793f56ac9..4527587709d 100644 --- a/python/src/nix-to-python.cc +++ b/python/src/nix-to-python.cc @@ -6,84 +6,83 @@ namespace pythonnix { -PyObject *nixToPythonObject(nix::EvalState &state, nix::Value &v, - nix::PathSet &context) { - switch (v.type()) { - case nix::nInt: - return PyLong_FromLong(v.integer); +PyObject * nixToPythonObject(nix::EvalState & state, nix::Value & v, nix::PathSet & context) +{ + switch (v.type()) { + case nix::nInt: + return PyLong_FromLong(v.integer); - case nix::nBool: - if (v.boolean) { - Py_RETURN_TRUE; - } else { - Py_RETURN_FALSE; - } - case nix::nString: - copyContext(v, context); - return PyUnicode_FromString(v.string.s); + case nix::nBool: + if (v.boolean) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } + case nix::nString: + copyContext(v, context); + return PyUnicode_FromString(v.string.s); - case nix::nPath: { - auto p = state.copyPathToStore(context, v.path).to_string(); - return PyUnicode_FromStringAndSize(p.data(), p.length()); - } + case nix::nPath: { + auto p = state.copyPathToStore(context, v.path).to_string(); + return PyUnicode_FromStringAndSize(p.data(), p.length()); + } - case nix::nNull: - Py_RETURN_NONE; + case nix::nNull: + Py_RETURN_NONE; - case nix::nAttrs: { - auto i = v.attrs->find(state.sOutPath); - if (i == v.attrs->end()) { - PyObjPtr dict(PyDict_New()); - if (!dict) { - return (PyObject *)nullptr; - } + case nix::nAttrs: { + auto i = v.attrs->find(state.sOutPath); + if (i == v.attrs->end()) { + PyObjPtr dict(PyDict_New()); + if (!dict) { + return (PyObject *) nullptr; + } - for (auto &j : *v.attrs) { - const std::string & name = state.symbols[j.name]; - auto value = nixToPythonObject(state, *j.value, context); - if (!value) { - return nullptr; + for (auto & j : *v.attrs) { + const std::string & name = state.symbols[j.name]; + auto value = nixToPythonObject(state, *j.value, context); + if (!value) { + return nullptr; + } + PyDict_SetItemString(dict.get(), name.c_str(), value); + } + return dict.release(); + } else { + return nixToPythonObject(state, *i->value, context); } - PyDict_SetItemString(dict.get(), name.c_str(), value); - } - return dict.release(); - } else { - return nixToPythonObject(state, *i->value, context); } - } - case nix::nList: { - PyObjPtr list(PyList_New(v.listSize())); - if (!list) { - return (PyObject *)nullptr; - } + case nix::nList: { + PyObjPtr list(PyList_New(v.listSize())); + if (!list) { + return (PyObject *) nullptr; + } - for (unsigned int n = 0; n < v.listSize(); ++n) { - auto value = nixToPythonObject(state, *v.listElems()[n], context); - if (!value) { - return nullptr; - } - PyList_SET_ITEM(list.get(), n, value); + for (unsigned int n = 0; n < v.listSize(); ++n) { + auto value = nixToPythonObject(state, *v.listElems()[n], context); + if (!value) { + return nullptr; + } + PyList_SET_ITEM(list.get(), n, value); + } + return list.release(); } - return list.release(); - } - case nix::nExternal: - return PyUnicode_FromString(""); + case nix::nExternal: + return PyUnicode_FromString(""); - case nix::nThunk: - return PyUnicode_FromString(""); + case nix::nThunk: + return PyUnicode_FromString(""); - case nix::nFunction: - return PyUnicode_FromString(""); + case nix::nFunction: + return PyUnicode_FromString(""); - case nix::nFloat: - return PyFloat_FromDouble(v.fpoint); + case nix::nFloat: + return PyFloat_FromDouble(v.fpoint); - default: - PyErr_Format(NixError, "cannot convert nix type '%s' to a python object", - showType(v).c_str()); - return nullptr; - } + default: + PyErr_Format(NixError, "cannot convert nix type '%s' to a python object", showType(v).c_str()); + return nullptr; + } } } // namespace pythonnix diff --git a/python/src/python-module.cc b/python/src/python-module.cc index 25806929bdb..f255abc0f9c 100644 --- a/python/src/python-module.cc +++ b/python/src/python-module.cc @@ -13,12 +13,10 @@ namespace pythonnix { #define _public_ __attribute__((visibility("default"))) -PyObject *NixError = nullptr; +PyObject * NixError = nullptr; -static PyMethodDef NixMethods[] = {{"eval", (PyCFunction)eval, - METH_VARARGS | METH_KEYWORDS, - "Eval nix expression"}, - {NULL, NULL, 0, NULL}}; +static PyMethodDef NixMethods[] = { + {"eval", (PyCFunction) eval, METH_VARARGS | METH_KEYWORDS, "Eval nix expression"}, {NULL, NULL, 0, NULL}}; static struct PyModuleDef nixmodule = { PyModuleDef_HEAD_INIT, "nix", "Nix expression bindings", @@ -26,36 +24,38 @@ static struct PyModuleDef nixmodule = { or -1 if the module keeps state in global variables. */ NixMethods}; -extern "C" _public_ PyObject *PyInit_nix(void) { - // By default, Nix sets the build-hook to be "$(readlink /proc/self/exe) __build-remote", expecting the current binary to be Nix itself. - // But when we call the Nix library from Python this isn't the case, the current binary is Python then - // So we need to change this default, pointing it to the Nix binary instead - nix::settings.buildHook = nix::settings.nixBinDir + "/nix __build-remote"; - // And by setting buildHook before calling initNix, we can override the defaults without overriding the user-provided options from the config files - nix::initNix(); - nix::initGC(); - - PyObjPtr m(PyModule_Create(&nixmodule)); - - if (!m) { - return nullptr; - } - - NixError = PyErr_NewExceptionWithDoc( - "nix.NixError", /* char *name */ - "Base exception class for the nix module.", /* char *doc */ - NULL, /* PyObject *base */ - NULL /* PyObject *dict */ - ); - - if (!NixError) { - return nullptr; - } - - if (PyModule_AddObject(m.get(), "NixError", NixError) == -1) { - return nullptr; - } - - return m.release(); +extern "C" _public_ PyObject * PyInit_nix(void) +{ + // By default, Nix sets the build-hook to be "$(readlink /proc/self/exe) __build-remote", expecting the current + // binary to be Nix itself. But when we call the Nix library from Python this isn't the case, the current binary is + // Python then So we need to change this default, pointing it to the Nix binary instead + nix::settings.buildHook = nix::settings.nixBinDir + "/nix __build-remote"; + // And by setting buildHook before calling initNix, we can override the defaults without overriding the + // user-provided options from the config files + nix::initNix(); + nix::initGC(); + + PyObjPtr m(PyModule_Create(&nixmodule)); + + if (!m) { + return nullptr; + } + + NixError = PyErr_NewExceptionWithDoc( + "nix.NixError", /* char *name */ + "Base exception class for the nix module.", /* char *doc */ + NULL, /* PyObject *base */ + NULL /* PyObject *dict */ + ); + + if (!NixError) { + return nullptr; + } + + if (PyModule_AddObject(m.get(), "NixError", NixError) == -1) { + return nullptr; + } + + return m.release(); } } // namespace pythonnix diff --git a/python/src/python-to-nix.cc b/python/src/python-to-nix.cc index f6326eef488..6f52187ab22 100644 --- a/python/src/python-to-nix.cc +++ b/python/src/python-to-nix.cc @@ -8,153 +8,157 @@ namespace pythonnix { -static const char *checkNullByte(const char *str, const Py_ssize_t size) { - for (Py_ssize_t i = 0; i < size; i++) { - if (str[0] == '\0') { - PyErr_Format(NixError, "invalid character: nix strings are not allowed " - "to contain null bytes"); - return nullptr; +static const char * checkNullByte(const char * str, const Py_ssize_t size) +{ + for (Py_ssize_t i = 0; i < size; i++) { + if (str[0] == '\0') { + PyErr_Format( + NixError, + "invalid character: nix strings are not allowed " + "to contain null bytes"); + return nullptr; + } } - } - return str; + return str; } -static const char *checkAttrKey(PyObject *obj) { - Py_ssize_t size = 0; - - if (!PyUnicode_Check(obj)) { - PyObjPtr typeName(PyObject_Str(PyObject_Type(obj))); - if (!typeName) { - return nullptr; +static const char * checkAttrKey(PyObject * obj) +{ + Py_ssize_t size = 0; + + if (!PyUnicode_Check(obj)) { + PyObjPtr typeName(PyObject_Str(PyObject_Type(obj))); + if (!typeName) { + return nullptr; + } + auto utf8 = PyUnicode_AsUTF8AndSize(typeName.get(), &size); + if (!utf8) { + return nullptr; + } + PyErr_Format(NixError, "key of nix attrsets must be strings, got type: %s", utf8); + return nullptr; } - auto utf8 = PyUnicode_AsUTF8AndSize(typeName.get(), &size); + + auto utf8 = PyUnicode_AsUTF8AndSize(obj, &size); if (!utf8) { - return nullptr; + return nullptr; } - PyErr_Format(NixError, "key of nix attrsets must be strings, got type: %s", - utf8); - return nullptr; - } - - auto utf8 = PyUnicode_AsUTF8AndSize(obj, &size); - if (!utf8) { - return nullptr; - } - return checkNullByte(utf8, size); + return checkNullByte(utf8, size); } -static std::optional dictToAttrSet(PyObject *obj, - nix::EvalState &state) { - PyObject *key = nullptr, *val = nullptr; - Py_ssize_t pos = 0; - - nix::ValueMap attrs; - while (PyDict_Next(obj, &pos, &key, &val)) { - auto name = checkAttrKey(key); - if (!name) { - return {}; - } - - auto attrVal = pythonToNixValue(state, val); - if (!attrVal) { - return {}; +static std::optional dictToAttrSet(PyObject * obj, nix::EvalState & state) +{ + PyObject *key = nullptr, *val = nullptr; + Py_ssize_t pos = 0; + + nix::ValueMap attrs; + while (PyDict_Next(obj, &pos, &key, &val)) { + auto name = checkAttrKey(key); + if (!name) { + return {}; + } + + auto attrVal = pythonToNixValue(state, val); + if (!attrVal) { + return {}; + } + attrs[state.symbols.create(name)] = attrVal; } - attrs[state.symbols.create(name)] = attrVal; - } - return attrs; + return attrs; } -nix::Value *pythonToNixValue(nix::EvalState &state, PyObject *obj) { - auto v = state.allocValue(); - - if (obj == Py_True && obj == Py_False) { - v->mkBool(obj == Py_True); - } else if (obj == Py_None) { - v->mkNull(); - } else if (PyBytes_Check(obj)) { - auto str = checkNullByte(PyBytes_AS_STRING(obj), PyBytes_GET_SIZE(obj)); - if (!str) { - return nullptr; - } - - v->mkString(str); - } else if (PyUnicode_Check(obj)) { - Py_ssize_t size; - const char *utf8 = PyUnicode_AsUTF8AndSize(obj, &size); - auto str = checkNullByte(utf8, size); - if (!str) { - return nullptr; +nix::Value * pythonToNixValue(nix::EvalState & state, PyObject * obj) +{ + auto v = state.allocValue(); + + if (obj == Py_True && obj == Py_False) { + v->mkBool(obj == Py_True); + } else if (obj == Py_None) { + v->mkNull(); + } else if (PyBytes_Check(obj)) { + auto str = checkNullByte(PyBytes_AS_STRING(obj), PyBytes_GET_SIZE(obj)); + if (!str) { + return nullptr; + } + + v->mkString(str); + } else if (PyUnicode_Check(obj)) { + Py_ssize_t size; + const char * utf8 = PyUnicode_AsUTF8AndSize(obj, &size); + auto str = checkNullByte(utf8, size); + if (!str) { + return nullptr; + } + + v->mkString(utf8); + } else if (PyFloat_Check(obj)) { + v->mkFloat(PyFloat_AS_DOUBLE(obj)); + } else if (PyLong_Check(obj)) { + v->mkInt(PyLong_AsLong(obj)); + } else if (PyList_Check(obj)) { + v->mkList(PyList_GET_SIZE(obj)); + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(obj); i++) { + auto val = pythonToNixValue(state, PyList_GET_ITEM(obj, i)); + if (!val) { + return nullptr; + } + v->listElems()[i] = val; + } + } else if (PyTuple_Check(obj)) { + v->mkList(PyTuple_GET_SIZE(obj)); + for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(obj); i++) { + auto val = pythonToNixValue(state, PyTuple_GET_ITEM(obj, i)); + if (!val) { + return nullptr; + } + v->listElems()[i] = val; + } + } else if (PyDict_Check(obj)) { + auto attrs = dictToAttrSet(obj, state); + if (!attrs) { + return nullptr; + } + auto attrsValue = attrs.value(); + auto bindings = state.buildBindings(attrsValue.size()); + for (auto & attr : attrsValue) { + bindings.insert(attr.first, attr.second); + } + v->mkAttrs(bindings); } - - v->mkString(utf8); - } else if (PyFloat_Check(obj)) { - v->mkFloat(PyFloat_AS_DOUBLE(obj)); - } else if (PyLong_Check(obj)) { - v->mkInt(PyLong_AsLong(obj)); - } else if (PyList_Check(obj)) { - v->mkList(PyList_GET_SIZE(obj)); - for (Py_ssize_t i = 0; i < PyList_GET_SIZE(obj); i++) { - auto val = pythonToNixValue(state, PyList_GET_ITEM(obj, i)); - if (!val) { - return nullptr; - } - v->listElems()[i] = val; - } - } else if (PyTuple_Check(obj)) { - v->mkList(PyTuple_GET_SIZE(obj)); - for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(obj); i++) { - auto val = pythonToNixValue(state, PyTuple_GET_ITEM(obj, i)); - if (!val) { - return nullptr; - } - v->listElems()[i] = val; - } - } else if (PyDict_Check(obj)) { - auto attrs = dictToAttrSet(obj, state); - if (!attrs) { - return nullptr; - } - auto attrsValue = attrs.value(); - auto bindings = state.buildBindings(attrsValue.size()); - for (auto &attr : attrsValue) { - bindings.insert(attr.first, attr.second); - } - v->mkAttrs(bindings); - } - return v; + return v; } -std::optional pythonToNixEnv(nix::EvalState &state, - PyObject *vars, nix::Env **env) { - Py_ssize_t pos = 0; - PyObject *key = nullptr, *val = nullptr; +std::optional pythonToNixEnv(nix::EvalState & state, PyObject * vars, nix::Env ** env) +{ + Py_ssize_t pos = 0; + PyObject *key = nullptr, *val = nullptr; - *env = &state.allocEnv(vars ? PyDict_Size(vars) : 0); - (*env)->up = &state.baseEnv; + *env = &state.allocEnv(vars ? PyDict_Size(vars) : 0); + (*env)->up = &state.baseEnv; - nix::StaticEnv staticEnv(false, state.staticBaseEnv.get()); + nix::StaticEnv staticEnv(false, state.staticBaseEnv.get()); - if (!vars) { - return staticEnv; - } - - auto displ = 0; - while (PyDict_Next(vars, &pos, &key, &val)) { - auto name = checkAttrKey(key); - if (!name) { - return {}; + if (!vars) { + return staticEnv; } - auto attrVal = pythonToNixValue(state, val); - if (!attrVal) { - return {}; + auto displ = 0; + while (PyDict_Next(vars, &pos, &key, &val)) { + auto name = checkAttrKey(key); + if (!name) { + return {}; + } + + auto attrVal = pythonToNixValue(state, val); + if (!attrVal) { + return {}; + } + staticEnv.vars.emplace_back(state.symbols.create(name), displ); + (*env)->values[displ++] = attrVal; } - staticEnv.vars.emplace_back(state.symbols.create(name), displ); - (*env)->values[displ++] = attrVal; - } - return staticEnv; + return staticEnv; } } // namespace pythonnix From 410393f0fd392fae44d7d4a5a6d918c7eea1a41c Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 17 Feb 2023 20:28:29 +0100 Subject: [PATCH 06/42] python: Rename pythonnix namespace to nix::python --- python/src/eval.cc | 4 ++-- python/src/internal/errors.hh | 2 +- python/src/internal/eval.hh | 2 +- python/src/internal/nix-to-python.hh | 4 ++-- python/src/internal/ptr.hh | 4 ++-- python/src/internal/python-to-nix.hh | 4 ++-- python/src/nix-to-python.cc | 4 ++-- python/src/python-module.cc | 4 ++-- python/src/python-to-nix.cc | 4 ++-- 9 files changed, 16 insertions(+), 16 deletions(-) diff --git a/python/src/eval.cc b/python/src/eval.cc index e95f7c88b96..d46f9aae2fe 100644 --- a/python/src/eval.cc +++ b/python/src/eval.cc @@ -7,7 +7,7 @@ #include #include -namespace pythonnix { +namespace nix::python { const char * currentExceptionTypeName() { @@ -58,4 +58,4 @@ PyObject * eval(PyObject * self, PyObject * args, PyObject * keywds) return PyErr_Format(NixError, "unexpected C++ exception: '%s'", currentExceptionTypeName()); } } -} // namespace pythonnix +} // namespace nix::python diff --git a/python/src/internal/errors.hh b/python/src/internal/errors.hh index a4f6bd81e0b..3934d70bea0 100644 --- a/python/src/internal/errors.hh +++ b/python/src/internal/errors.hh @@ -2,7 +2,7 @@ #include -namespace pythonnix { +namespace nix::python { extern PyObject * NixError; } diff --git a/python/src/internal/eval.hh b/python/src/internal/eval.hh index 878b82ff643..4a4861b7a57 100644 --- a/python/src/internal/eval.hh +++ b/python/src/internal/eval.hh @@ -2,7 +2,7 @@ #include -namespace pythonnix { +namespace nix::python { PyObject * eval(PyObject * self, PyObject * args, PyObject * kwdict); } diff --git a/python/src/internal/nix-to-python.hh b/python/src/internal/nix-to-python.hh index bfcc8bf22ba..50c22832528 100644 --- a/python/src/internal/nix-to-python.hh +++ b/python/src/internal/nix-to-python.hh @@ -6,7 +6,7 @@ #include -namespace pythonnix { +namespace nix::python { PyObject * nixToPythonObject(nix::EvalState & state, nix::Value & v, nix::PathSet & context); -} // namespace pythonnix +} // namespace nix::python diff --git a/python/src/internal/ptr.hh b/python/src/internal/ptr.hh index 5ca073de283..e723802ec0c 100644 --- a/python/src/internal/ptr.hh +++ b/python/src/internal/ptr.hh @@ -3,7 +3,7 @@ #include #include -namespace pythonnix { +namespace nix::python { struct PyObjectDeleter { @@ -14,4 +14,4 @@ struct PyObjectDeleter }; typedef std::unique_ptr PyObjPtr; -} // namespace pythonnix +} // namespace nix::python diff --git a/python/src/internal/python-to-nix.hh b/python/src/internal/python-to-nix.hh index 3137336e964..3f50234ccc0 100644 --- a/python/src/internal/python-to-nix.hh +++ b/python/src/internal/python-to-nix.hh @@ -6,9 +6,9 @@ #include #include -namespace pythonnix { +namespace nix::python { nix::Value * pythonToNixValue(nix::EvalState & state, PyObject * obj); std::optional pythonToNixEnv(nix::EvalState & state, PyObject * vars, nix::Env ** env); -} // namespace pythonnix +} // namespace nix::python diff --git a/python/src/nix-to-python.cc b/python/src/nix-to-python.cc index 4527587709d..7820c27e030 100644 --- a/python/src/nix-to-python.cc +++ b/python/src/nix-to-python.cc @@ -4,7 +4,7 @@ #include "internal/nix-to-python.hh" #include "internal/ptr.hh" -namespace pythonnix { +namespace nix::python { PyObject * nixToPythonObject(nix::EvalState & state, nix::Value & v, nix::PathSet & context) { @@ -85,4 +85,4 @@ PyObject * nixToPythonObject(nix::EvalState & state, nix::Value & v, nix::PathSe return nullptr; } } -} // namespace pythonnix +} // namespace nix::python diff --git a/python/src/python-module.cc b/python/src/python-module.cc index f255abc0f9c..9f933c81f0f 100644 --- a/python/src/python-module.cc +++ b/python/src/python-module.cc @@ -9,7 +9,7 @@ #include #include -namespace pythonnix { +namespace nix::python { #define _public_ __attribute__((visibility("default"))) @@ -58,4 +58,4 @@ extern "C" _public_ PyObject * PyInit_nix(void) return m.release(); } -} // namespace pythonnix +} // namespace nix::python diff --git a/python/src/python-to-nix.cc b/python/src/python-to-nix.cc index 6f52187ab22..af69731edeb 100644 --- a/python/src/python-to-nix.cc +++ b/python/src/python-to-nix.cc @@ -6,7 +6,7 @@ #include -namespace pythonnix { +namespace nix::python { static const char * checkNullByte(const char * str, const Py_ssize_t size) { @@ -161,4 +161,4 @@ std::optional pythonToNixEnv(nix::EvalState & state, PyObject * return staticEnv; } -} // namespace pythonnix +} // namespace nix::python From 8837e42ba39870ead114e9acf939df65b7817e29 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 17 Feb 2023 21:34:56 +0100 Subject: [PATCH 07/42] python: Add exampleEnv to try out the bindings Will be useful for documentation --- python/default.nix | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/python/default.nix b/python/default.nix index f5a3132d5e7..a4c68c3af65 100644 --- a/python/default.nix +++ b/python/default.nix @@ -36,10 +36,13 @@ python.pkgs.buildPythonPackage { doInstallCheck = true; installCheckPhase = "meson test -v"; - passthru.shell = mkShell { - inputsFrom = [ - self.devShells.${system}.default - (nix.python-bindings.override { nix = null; }) - ]; + passthru = { + exampleEnv = python.withPackages (p: [ nix.python-bindings ]); + shell = mkShell { + inputsFrom = [ + self.devShells.${system}.default + (nix.python-bindings.override { nix = null; }) + ]; + }; }; } From 4599be31a71e742e8253a91be9efd312c86c3e3e Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 17 Feb 2023 23:19:38 +0100 Subject: [PATCH 08/42] python: Install the bindings in hopefully the correct location And don't use the deprecated python3 meson module --- python/meson.build | 7 ++++--- python/src/meson.build | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/python/meson.build b/python/meson.build index 97ba97e7309..e2f9294c02d 100644 --- a/python/meson.build +++ b/python/meson.build @@ -3,8 +3,9 @@ project('python-nix', 'cpp', license : 'LGPL-2.0', ) -python_mod = import('python3') -python_dep = dependency('python3', required : true) +python_mod = import('python') +py_installation = python_mod.find_installation() + nix_expr_dep = dependency('nix-expr', required: true) nix_main_dep = dependency('nix-main', required: true) @@ -16,6 +17,6 @@ nix_root = fs.parent(meson.project_source_root()) run_command('make', '-C', nix_root, 'tests/common/vars-and-functions.sh', check: true) env = environment() -env.prepend('PYTHONPATH', fs.parent(pythonix.full_path())) +env.prepend('PYTHONPATH', fs.parent(nix_bindings.full_path())) bash = find_program('bash') test('python test', bash, args : files('test.sh'), env : env) diff --git a/python/src/meson.build b/python/src/meson.build index 937d755913b..814be713c84 100644 --- a/python/src/meson.build +++ b/python/src/meson.build @@ -5,10 +5,10 @@ src = [ 'python-module.cc', ] -pythonix = python_mod.extension_module('nix', src, - dependencies : [python_dep, nix_expr_dep, nix_main_dep], +nix_bindings = py_installation.extension_module('nix', src, + dependencies : [nix_expr_dep, nix_main_dep], install: true, - install_dir: python_mod.sysconfig_path('platlib'), + subdir: 'nix', cpp_args: [ '-std=c++17', # -Wnon-virtual-dtor is unnecessarily turned on by Meson From be36946e0a8e6f365ee36b00762db182052888bc Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 17 Feb 2023 23:38:06 +0100 Subject: [PATCH 09/42] WIP: Start writing documentation and example --- doc/manual/local.mk | 3 +- doc/manual/src/SUMMARY.md.in | 3 ++ doc/manual/src/python | 1 + python/README.md | 13 ++++++++ python/doc/api.md | 3 ++ python/doc/hacking.md | 3 ++ python/doc/index.md | 32 +++++++++++++++++++ .../buildPythonApplication/default.nix | 26 +++++++++++++++ .../buildPythonApplication/hello/__init__.py | 4 +++ .../examples/buildPythonApplication/setup.py | 13 ++++++++ 10 files changed, 100 insertions(+), 1 deletion(-) create mode 120000 doc/manual/src/python create mode 100644 python/README.md create mode 100644 python/doc/api.md create mode 100644 python/doc/hacking.md create mode 100644 python/doc/index.md create mode 100644 python/examples/buildPythonApplication/default.nix create mode 100644 python/examples/buildPythonApplication/hello/__init__.py create mode 100644 python/examples/buildPythonApplication/setup.py diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 63e7e61e437..808d9267c85 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -2,7 +2,8 @@ ifeq ($(doc_generate),yes) MANUAL_SRCS := \ $(call rwildcard, $(d)/src, *.md) \ - $(call rwildcard, $(d)/src, */*.md) + $(call rwildcard, $(d)/src, */*.md) \ + $(call rwildcard, $(d)/../../python/doc, *.md) man-pages := $(foreach n, \ nix-env.1 nix-store.1 \ diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index f783d590839..2c44741068d 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -92,6 +92,9 @@ - [Files](command-ref/files.md) - [nix.conf](command-ref/conf-file.md) - [Architecture](architecture/architecture.md) +- [Python Bindings](python/index.md) + - [API](python/api.md) + - [Hacking](python/hacking.md) - [Glossary](glossary.md) - [Contributing](contributing/contributing.md) - [Hacking](contributing/hacking.md) diff --git a/doc/manual/src/python b/doc/manual/src/python new file mode 120000 index 00000000000..14960743609 --- /dev/null +++ b/doc/manual/src/python @@ -0,0 +1 @@ +../../../python/doc \ No newline at end of file diff --git a/python/README.md b/python/README.md new file mode 100644 index 00000000000..886f9b6ebb3 --- /dev/null +++ b/python/README.md @@ -0,0 +1,13 @@ +# Python Bindings + +This directory contains experimental Python bindings to a small subset of Nix's functionality. These bindings are very fast since they link to the necessary dynamic libraries directly, without having to call the Nix CLI for every operation. + +Thanks to [@Mic92](https://github.com/Mic92) who wrote [Pythonix](https://github.com/Mic92/pythonix) which these bindings were originally based on, before they became the official bindings that are part of the Nix project. They were upstreamed to decrease maintenance overhead and make sure they are always up-to-date. + +Note that the Python bindings are new and experimental. The interface is likely to change based on known issues and user feedback. + +## Documentation + +See [index.md](./doc/index.md), which is also rendered in the HTML manual. + +To hack on these bindings, see [hacking.md](./doc/hacking.md), also rendered in the HTML manual. diff --git a/python/doc/api.md b/python/doc/api.md new file mode 100644 index 00000000000..e586330cd13 --- /dev/null +++ b/python/doc/api.md @@ -0,0 +1,3 @@ +# Experimental Python Bindings API + +This is the API! diff --git a/python/doc/hacking.md b/python/doc/hacking.md new file mode 100644 index 00000000000..dff7e4d5e75 --- /dev/null +++ b/python/doc/hacking.md @@ -0,0 +1,3 @@ +# Python Bindings Hacking + +This is how to hack on the bindings diff --git a/python/doc/index.md b/python/doc/index.md new file mode 100644 index 00000000000..53b29fff242 --- /dev/null +++ b/python/doc/index.md @@ -0,0 +1,32 @@ +# Experimental Python Bindings + +Nix comes with minimal experimental Python bindings that link directly to the necessary dynamic libraries, making them very fast. + +## Trying it out + +The easiest way to try out the bindings is using the provided example environment: + +``` +$ nix run github:NixOS/nix#nix.python-bindings.exampleEnv +Python 3.10.8 (main, Oct 11 2022, 11:35:05) [GCC 11.3.0] on linux +Type "help", "copyright", "credits" or "license" for more information. +>>> import nix +>>> nix.eval('"Hello ${name}!"', vars=dict(name="Python")) +'Hello Python!' +``` + +For the available functions and their interfaces, see the API section. + +## Build integration + +In the future these Python bindings will be available from Nixpkgs as `python3Packages.nix`. + +Until then the Python bindings are only available from the Nix derivation via the `python-bindings` [passthru attribute](https://nixos.org/manual/nixpkgs/stable/#var-stdenv-passthru). Without any modifications, this derivation is built for the default Python 3 version from the Nixpkgs version used to build Nix. This Python version might not match the Python version of the project you're trying to use them in. Therefore it is recommended to override the bindings with the correct Python version using + +``` +nix.python-bindings.override { + python = myPythonVersion; +} +``` + +For complete examples, see https://github.com/NixOS/nix/tree/master/python/examples diff --git a/python/examples/buildPythonApplication/default.nix b/python/examples/buildPythonApplication/default.nix new file mode 100644 index 00000000000..88c9c390e30 --- /dev/null +++ b/python/examples/buildPythonApplication/default.nix @@ -0,0 +1,26 @@ +{ + nixpkgsSrc ? fetchTarball "https://github.com/NixOS/nixpkgs/archive/545c7a31e5dedea4a6d372712a18e00ce097d462.tar.gz", + nixSrc ? fetchTarball "https://github.com/tweag/nix/archive/9b49f6902c5511e96a2a822bc36f245caf78de3f.tar.gz", +}: let + pkgs = import nixpkgsSrc { overlays = []; config = {}; }; + nix = (import nixSrc).default; + python = pkgs.python3; + nixBindings = nix.python-bindings.override { inherit python; }; +in python.pkgs.buildPythonApplication { + pname = "hello-nix"; + version = "0.1"; + src = ./.; + propagatedBuildInputs = [ + nixBindings + #python.pkgs.requests + ]; + #preBuild = '' + # echo $PYTHONPATH + # exit 1 + #''; + dontUseSetuptoolsShellHook = true; + passthru.nixBindings = nixBindings; + #checkPhase = '' + # ls -laa $out/hello-nix + #''; +} diff --git a/python/examples/buildPythonApplication/hello/__init__.py b/python/examples/buildPythonApplication/hello/__init__.py new file mode 100644 index 00000000000..786e4327b31 --- /dev/null +++ b/python/examples/buildPythonApplication/hello/__init__.py @@ -0,0 +1,4 @@ +import nix + +def greet(): + print("Evaluating 1 + 1 in Nix gives:" + nix.eval("1 + 1")) diff --git a/python/examples/buildPythonApplication/setup.py b/python/examples/buildPythonApplication/setup.py new file mode 100644 index 00000000000..3fde1535e9b --- /dev/null +++ b/python/examples/buildPythonApplication/setup.py @@ -0,0 +1,13 @@ +from setuptools import setup, find_packages + +setup( + name="hello-nix", + version="0.1", + packages=find_packages(), + entry_points={ + 'console_scripts': [ + 'hello-nix = hello:greet', + ], + }, + #install_requires=['nix'], +) From e3f56e9deda7d84509cc6b6f4ccad55cb70c5b85 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 24 Feb 2023 13:22:04 +0100 Subject: [PATCH 10/42] Remove .cache from .gitignore Not sure why I added it originally --- python/.gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/python/.gitignore b/python/.gitignore index 7194ea7277e..378eac25d31 100644 --- a/python/.gitignore +++ b/python/.gitignore @@ -1,2 +1 @@ -.cache build From 2c3e1c88da8d94f01cac8dfce116c57506d3b59c Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 24 Feb 2023 13:22:30 +0100 Subject: [PATCH 11/42] Always include config.h Mirroring what the main Nix build does --- python/src/internal/nix-to-python.hh | 2 -- python/src/internal/python-to-nix.hh | 3 +-- python/src/meson.build | 3 ++- python/src/python-module.cc | 2 -- 4 files changed, 3 insertions(+), 7 deletions(-) diff --git a/python/src/internal/nix-to-python.hh b/python/src/internal/nix-to-python.hh index 50c22832528..997a1e7adcb 100644 --- a/python/src/internal/nix-to-python.hh +++ b/python/src/internal/nix-to-python.hh @@ -2,8 +2,6 @@ #include -#include - #include namespace nix::python { diff --git a/python/src/internal/python-to-nix.hh b/python/src/internal/python-to-nix.hh index 3f50234ccc0..e922323893b 100644 --- a/python/src/internal/python-to-nix.hh +++ b/python/src/internal/python-to-nix.hh @@ -1,7 +1,6 @@ #pragma once -#include -#include +struct PyObject; #include #include diff --git a/python/src/meson.build b/python/src/meson.build index 814be713c84..1caf34b0a90 100644 --- a/python/src/meson.build +++ b/python/src/meson.build @@ -14,5 +14,6 @@ nix_bindings = py_installation.extension_module('nix', src, # -Wnon-virtual-dtor is unnecessarily turned on by Meson # This is fixed in Meson 1.0.0 with https://github.com/mesonbuild/meson/pull/10339 '-Wno-non-virtual-dtor', - '-fvisibility=hidden' + '-fvisibility=hidden', + '-include', 'config.h' ]) diff --git a/python/src/python-module.cc b/python/src/python-module.cc index 9f933c81f0f..d2d8b4b349d 100644 --- a/python/src/python-module.cc +++ b/python/src/python-module.cc @@ -3,8 +3,6 @@ #include "internal/eval.hh" #include "internal/ptr.hh" -#include - #include #include #include From 9b27833252248fb9e5e992cca84c8b28b004e805 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 24 Feb 2023 13:22:52 +0100 Subject: [PATCH 12/42] Release the GIL for Nix evaluation --- python/src/eval.cc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/python/src/eval.cc b/python/src/eval.cc index d46f9aae2fe..31a94beb5b1 100644 --- a/python/src/eval.cc +++ b/python/src/eval.cc @@ -28,11 +28,18 @@ static PyObject * _eval(const char * expression, PyObject * vars) } auto staticEnvPointer = std::make_shared(*staticEnv); + // Release the GIL, so that other Python threads can be running in parallel + // while the potentially expensive Nix evaluation happens. This is safe + // because we don't operate on Python objects or call the Python/C API in + // this block + // See https://docs.python.org/3/c-api/init.html#thread-state-and-the-global-interpreter-lock + Py_BEGIN_ALLOW_THREADS auto e = state.parseExprFromString(expression, ".", staticEnvPointer); nix::Value v; - e->eval(state, *env, v); + e->eval(state, *env, v); state.forceValueDeep(v); + Py_END_ALLOW_THREADS nix::PathSet context; return nixToPythonObject(state, v, context); From d27217162c1a7ace31a4530197c418f9baaac5c7 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 24 Feb 2023 19:05:05 +0100 Subject: [PATCH 13/42] Don't release GIL lock because it would segfault --- python/src/eval.cc | 9 +++++---- python/tests.py | 5 +++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/python/src/eval.cc b/python/src/eval.cc index 31a94beb5b1..268432c27ac 100644 --- a/python/src/eval.cc +++ b/python/src/eval.cc @@ -27,19 +27,20 @@ static PyObject * _eval(const char * expression, PyObject * vars) return nullptr; } auto staticEnvPointer = std::make_shared(*staticEnv); + nix::Value v; + + // FIXME: Doing this would breaks the test_GIL_case test case // Release the GIL, so that other Python threads can be running in parallel // while the potentially expensive Nix evaluation happens. This is safe // because we don't operate on Python objects or call the Python/C API in // this block // See https://docs.python.org/3/c-api/init.html#thread-state-and-the-global-interpreter-lock - Py_BEGIN_ALLOW_THREADS + // Py_BEGIN_ALLOW_THREADS auto e = state.parseExprFromString(expression, ".", staticEnvPointer); - nix::Value v; - e->eval(state, *env, v); state.forceValueDeep(v); - Py_END_ALLOW_THREADS + // Py_END_ALLOW_THREADS nix::PathSet context; return nixToPythonObject(state, v, context); diff --git a/python/tests.py b/python/tests.py index d2c4b1301ab..5e523f23dcb 100644 --- a/python/tests.py +++ b/python/tests.py @@ -27,5 +27,10 @@ def test_ifd(self): """ self.assertEqual(nix.eval(expression, vars=dict()), "test") + # This test case fails if you uncomment the `Py_{BEGIN,END}_ALLOW_THREADS` + # macros in src/eval.cc + def test_GIL_case(self): + nix.eval("{ a = throw \"nope\"; }") + if __name__ == '__main__': unittest.main() From b8003d6c495da4bc82458bc43de543eed70c0ff8 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 24 Feb 2023 19:05:46 +0100 Subject: [PATCH 14/42] Allow getting exact throw error messages --- python/src/eval.cc | 3 +++ python/tests.py | 12 +++++++++++- src/libutil/error.hh | 4 ++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/python/src/eval.cc b/python/src/eval.cc index 268432c27ac..f0eca2b4a7c 100644 --- a/python/src/eval.cc +++ b/python/src/eval.cc @@ -60,6 +60,9 @@ PyObject * eval(PyObject * self, PyObject * args, PyObject * keywds) try { return _eval(expression, vars); + } catch (nix::ThrownError & e) { + return PyErr_Format(NixError, "%s", e.message().c_str()); + } catch (nix::Error & e) { return PyErr_Format(NixError, "%s", e.what()); } catch (...) { diff --git a/python/tests.py b/python/tests.py index 5e523f23dcb..d54130f34e4 100644 --- a/python/tests.py +++ b/python/tests.py @@ -27,10 +27,20 @@ def test_ifd(self): """ self.assertEqual(nix.eval(expression, vars=dict()), "test") + def test_throw(self): + errorString = "hello hi there\ntest" + try: + nix.eval("throw str", vars=dict(str=errorString)) + except nix.NixError as e: + self.assertEqual(e.args[0], errorString) + # This test case fails if you uncomment the `Py_{BEGIN,END}_ALLOW_THREADS` # macros in src/eval.cc def test_GIL_case(self): - nix.eval("{ a = throw \"nope\"; }") + try: + nix.eval("{ a = throw \"nope\"; }") + except nix.NixError as e: + self.assertEqual(e.args[0], "nope") if __name__ == '__main__': unittest.main() diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 6a09230812e..d3aa5eccbe3 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -154,6 +154,10 @@ public: : err(e) { } + std::string message() { + return err.msg.str(); + } + #ifdef EXCEPTION_NEEDS_THROW_SPEC ~BaseError() throw () { }; const char * what() const throw () { return calcWhat().c_str(); } From a1669ed32989f22f551374e36468bb528b98d207 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 24 Feb 2023 19:06:22 +0100 Subject: [PATCH 15/42] Readd .cache to .gitignore --- python/.gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/.gitignore b/python/.gitignore index 378eac25d31..6cf006aad99 100644 --- a/python/.gitignore +++ b/python/.gitignore @@ -1 +1,3 @@ build +# For clang-tools +.cache From 33ca7e3ef90bd0cb9837195507b4bcce521f1d92 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Mon, 27 Feb 2023 14:38:08 +0100 Subject: [PATCH 16/42] Add ThrownNixError subclass --- python/src/eval.cc | 3 +-- python/src/internal/errors.hh | 1 + python/src/python-module.cc | 16 ++++++++++++++++ python/tests.py | 10 +++++++++- 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/python/src/eval.cc b/python/src/eval.cc index f0eca2b4a7c..9b32468b4a9 100644 --- a/python/src/eval.cc +++ b/python/src/eval.cc @@ -61,8 +61,7 @@ PyObject * eval(PyObject * self, PyObject * args, PyObject * keywds) try { return _eval(expression, vars); } catch (nix::ThrownError & e) { - return PyErr_Format(NixError, "%s", e.message().c_str()); - + return PyErr_Format(ThrownNixError, "%s", e.message().c_str()); } catch (nix::Error & e) { return PyErr_Format(NixError, "%s", e.what()); } catch (...) { diff --git a/python/src/internal/errors.hh b/python/src/internal/errors.hh index 3934d70bea0..da5ebef6192 100644 --- a/python/src/internal/errors.hh +++ b/python/src/internal/errors.hh @@ -5,4 +5,5 @@ namespace nix::python { extern PyObject * NixError; +extern PyObject * ThrownNixError; } diff --git a/python/src/python-module.cc b/python/src/python-module.cc index d2d8b4b349d..3c297550a26 100644 --- a/python/src/python-module.cc +++ b/python/src/python-module.cc @@ -11,6 +11,7 @@ namespace nix::python { #define _public_ __attribute__((visibility("default"))) +PyObject * ThrownNixError = nullptr; PyObject * NixError = nullptr; static PyMethodDef NixMethods[] = { @@ -54,6 +55,21 @@ extern "C" _public_ PyObject * PyInit_nix(void) return nullptr; } + ThrownNixError = PyErr_NewExceptionWithDoc( + "nix.ThrownNixError", /* char *name */ + "Base exception class for the nix module.", /* char *doc */ + NixError, /* PyObject *base */ + NULL /* PyObject *dict */ + ); + + if (!ThrownNixError) { + return nullptr; + } + + if (PyModule_AddObject(m.get(), "ThrownNixError", ThrownNixError) == -1) { + return nullptr; + } + return m.release(); } } // namespace nix::python diff --git a/python/tests.py b/python/tests.py index d54130f34e4..f9772bab8ba 100644 --- a/python/tests.py +++ b/python/tests.py @@ -31,9 +31,17 @@ def test_throw(self): errorString = "hello hi there\ntest" try: nix.eval("throw str", vars=dict(str=errorString)) - except nix.NixError as e: + except nix.ThrownNixError as e: self.assertEqual(e.args[0], errorString) + def test_syntax_error(self): + try: + nix.eval("{") + except nix.ThrownNixError as e: + self.assertTrue(False) + except nix.NixError as e: + self.assertTrue(True) + # This test case fails if you uncomment the `Py_{BEGIN,END}_ALLOW_THREADS` # macros in src/eval.cc def test_GIL_case(self): From 7dee1c5b03c34e7c9c6dd490d08f7efaaaff6369 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 3 Mar 2023 14:21:12 +0100 Subject: [PATCH 17/42] python: Add clang-tools to dev env --- python/default.nix | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/default.nix b/python/default.nix index a4c68c3af65..a05726b6f0d 100644 --- a/python/default.nix +++ b/python/default.nix @@ -1,4 +1,4 @@ -{ self, system, lib, python, ninja, meson, nix, mkShell }: +{ self, system, lib, python, clang-tools, ninja, meson, nix, mkShell }: python.pkgs.buildPythonPackage { name = "nix"; format = "other"; @@ -39,6 +39,9 @@ python.pkgs.buildPythonPackage { passthru = { exampleEnv = python.withPackages (p: [ nix.python-bindings ]); shell = mkShell { + packages = [ + clang-tools + ]; inputsFrom = [ self.devShells.${system}.default (nix.python-bindings.override { nix = null; }) From 5bc4948d8dddd399d0b0f29af7ac043942f09fa0 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 3 Mar 2023 14:21:31 +0100 Subject: [PATCH 18/42] python: Properly release and reacquire GIL --- python/src/eval.cc | 18 ++++++++++++------ python/tests.py | 2 -- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/python/src/eval.cc b/python/src/eval.cc index 9b32468b4a9..57513767b7c 100644 --- a/python/src/eval.cc +++ b/python/src/eval.cc @@ -6,6 +6,7 @@ #include #include +#include namespace nix::python { @@ -30,17 +31,22 @@ static PyObject * _eval(const char * expression, PyObject * vars) nix::Value v; - // FIXME: Doing this would breaks the test_GIL_case test case // Release the GIL, so that other Python threads can be running in parallel // while the potentially expensive Nix evaluation happens. This is safe // because we don't operate on Python objects or call the Python/C API in // this block // See https://docs.python.org/3/c-api/init.html#thread-state-and-the-global-interpreter-lock - // Py_BEGIN_ALLOW_THREADS - auto e = state.parseExprFromString(expression, ".", staticEnvPointer); - e->eval(state, *env, v); - state.forceValueDeep(v); - // Py_END_ALLOW_THREADS + { + PyThreadState *_save; + _save = PyEval_SaveThread(); + Finally reacquireGIL([&] { + PyEval_RestoreThread(_save); + }); + + auto e = state.parseExprFromString(expression, ".", staticEnvPointer); + e->eval(state, *env, v); + state.forceValueDeep(v); + } nix::PathSet context; return nixToPythonObject(state, v, context); diff --git a/python/tests.py b/python/tests.py index f9772bab8ba..7614b626480 100644 --- a/python/tests.py +++ b/python/tests.py @@ -42,8 +42,6 @@ def test_syntax_error(self): except nix.NixError as e: self.assertTrue(True) - # This test case fails if you uncomment the `Py_{BEGIN,END}_ALLOW_THREADS` - # macros in src/eval.cc def test_GIL_case(self): try: nix.eval("{ a = throw \"nope\"; }") From 6f8108c5f9e14076a1ae0ddc9678f322642ded93 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 3 Mar 2023 19:23:05 +0100 Subject: [PATCH 19/42] python: Detect stack overflow from recursive data structure --- python/src/internal/nix-to-python.hh | 1 + python/src/nix-to-python.cc | 21 +++++++++++++++++---- python/tests.py | 6 ++++++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/python/src/internal/nix-to-python.hh b/python/src/internal/nix-to-python.hh index 997a1e7adcb..809ff5f3ad1 100644 --- a/python/src/internal/nix-to-python.hh +++ b/python/src/internal/nix-to-python.hh @@ -7,4 +7,5 @@ namespace nix::python { PyObject * nixToPythonObject(nix::EvalState & state, nix::Value & v, nix::PathSet & context); +PyObject * _nixToPythonObject(nix::EvalState & state, nix::Value & v, nix::PathSet & context, std::set seen); } // namespace nix::python diff --git a/python/src/nix-to-python.cc b/python/src/nix-to-python.cc index 7820c27e030..fec51bf8820 100644 --- a/python/src/nix-to-python.cc +++ b/python/src/nix-to-python.cc @@ -6,7 +6,12 @@ namespace nix::python { -PyObject * nixToPythonObject(nix::EvalState & state, nix::Value & v, nix::PathSet & context) +PyObject * nixToPythonObject(nix::EvalState & state, nix::Value & v, nix::PathSet & context) { + std::set seen; + return _nixToPythonObject(state, v, context, seen); +} + +PyObject * _nixToPythonObject(nix::EvalState & state, nix::Value & v, nix::PathSet & context, std::set seen) { switch (v.type()) { case nix::nInt: @@ -31,6 +36,10 @@ PyObject * nixToPythonObject(nix::EvalState & state, nix::Value & v, nix::PathSe Py_RETURN_NONE; case nix::nAttrs: { + if (!v.attrs->empty() && !seen.insert(v.attrs).second) { + PyErr_Format(NixError, "Infinite recursion in data structure"); + return nullptr; + } auto i = v.attrs->find(state.sOutPath); if (i == v.attrs->end()) { PyObjPtr dict(PyDict_New()); @@ -40,7 +49,7 @@ PyObject * nixToPythonObject(nix::EvalState & state, nix::Value & v, nix::PathSe for (auto & j : *v.attrs) { const std::string & name = state.symbols[j.name]; - auto value = nixToPythonObject(state, *j.value, context); + auto value = _nixToPythonObject(state, *j.value, context, seen); if (!value) { return nullptr; } @@ -48,18 +57,22 @@ PyObject * nixToPythonObject(nix::EvalState & state, nix::Value & v, nix::PathSe } return dict.release(); } else { - return nixToPythonObject(state, *i->value, context); + return _nixToPythonObject(state, *i->value, context, seen); } } case nix::nList: { + if (v.listSize() && !seen.insert(v.listElems()).second) { + PyErr_Format(NixError, "Infinite recursion in data structure"); + return nullptr; + } PyObjPtr list(PyList_New(v.listSize())); if (!list) { return (PyObject *) nullptr; } for (unsigned int n = 0; n < v.listSize(); ++n) { - auto value = nixToPythonObject(state, *v.listElems()[n], context); + auto value = _nixToPythonObject(state, *v.listElems()[n], context, seen); if (!value) { return nullptr; } diff --git a/python/tests.py b/python/tests.py index 7614b626480..8e80d3ddcc0 100644 --- a/python/tests.py +++ b/python/tests.py @@ -48,5 +48,11 @@ def test_GIL_case(self): except nix.NixError as e: self.assertEqual(e.args[0], "nope") + def test_infinity(self): + try: + nix.eval("let x = { inherit x; }; in x") + except nix.NixError as e: + self.assertTrue(True) + if __name__ == '__main__': unittest.main() From 446db64f2ea4b040967c4cbc8a74bd90c1d7508e Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 3 Mar 2023 21:34:50 +0100 Subject: [PATCH 20/42] Insert some TODO's --- python/src/eval.cc | 1 + python/src/python-to-nix.cc | 1 + 2 files changed, 2 insertions(+) diff --git a/python/src/eval.cc b/python/src/eval.cc index 57513767b7c..d576049cefe 100644 --- a/python/src/eval.cc +++ b/python/src/eval.cc @@ -43,6 +43,7 @@ static PyObject * _eval(const char * expression, PyObject * vars) PyEval_RestoreThread(_save); }); + // TODO: Should the "." be something else here? auto e = state.parseExprFromString(expression, ".", staticEnvPointer); e->eval(state, *env, v); state.forceValueDeep(v); diff --git a/python/src/python-to-nix.cc b/python/src/python-to-nix.cc index af69731edeb..cc363abf5da 100644 --- a/python/src/python-to-nix.cc +++ b/python/src/python-to-nix.cc @@ -78,6 +78,7 @@ nix::Value * pythonToNixValue(nix::EvalState & state, PyObject * obj) } else if (obj == Py_None) { v->mkNull(); } else if (PyBytes_Check(obj)) { + // TODO: Bytes should probably not be coerced to strings auto str = checkNullByte(PyBytes_AS_STRING(obj), PyBytes_GET_SIZE(obj)); if (!str) { return nullptr; From 481f28c158287a298ebe2e4187850efb46efe694 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 3 Mar 2023 21:38:47 +0100 Subject: [PATCH 21/42] python: Handle null's in expressions correctly And test that null bytes currently end strings --- python/src/eval.cc | 15 ++++++++++++--- python/tests.py | 12 ++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/python/src/eval.cc b/python/src/eval.cc index d576049cefe..9ad3b44b910 100644 --- a/python/src/eval.cc +++ b/python/src/eval.cc @@ -17,7 +17,7 @@ const char * currentExceptionTypeName() return res ? res : "(null)"; } -static PyObject * _eval(const char * expression, PyObject * vars) +static PyObject * _eval(const std::string & expression, PyObject * vars) { nix::Strings storePath; nix::EvalState state(storePath, nix::openStore()); @@ -55,16 +55,25 @@ static PyObject * _eval(const char * expression, PyObject * vars) PyObject * eval(PyObject * self, PyObject * args, PyObject * keywds) { - const char * expression = nullptr; + PyObject * expressionObject; PyObject * vars = nullptr; const char * kwlist[] = {"expression", "vars", nullptr}; + // See https://docs.python.org/3/c-api/arg.html for the magic string if (!PyArg_ParseTupleAndKeywords( - args, keywds, "s|O!", const_cast(kwlist), &expression, &PyDict_Type, &vars)) { + args, keywds, "U|O!", const_cast(kwlist), &expressionObject, &PyDict_Type, &vars)) { return nullptr; } + // This handles null bytes in expressions correctly + Py_ssize_t expressionSize; + auto expressionBase = PyUnicode_AsUTF8AndSize(expressionObject, &expressionSize); + if (!expressionBase) { + return nullptr; + } + std::string expression(expressionBase, expressionSize); + try { return _eval(expression, vars); } catch (nix::ThrownError & e) { diff --git a/python/tests.py b/python/tests.py index 8e80d3ddcc0..3bb272aa1f5 100644 --- a/python/tests.py +++ b/python/tests.py @@ -54,5 +54,17 @@ def test_infinity(self): except nix.NixError as e: self.assertTrue(True) + def test_null_expression(self): + # Null characters should be allowed in expressions, even if they aren't + # very useful really, though at least null's should be supported in + # strings in the future https://github.com/NixOS/nix/issues/1307) + self.assertEqual(nix.eval("\"ab\x00cd\""), "ab") + + def test_throw_null(self): + try: + nix.eval("throw \"hello\x00there\"") + except nix.ThrownNixError as e: + self.assertEqual(e.args[0], "hello") + if __name__ == '__main__': unittest.main() From 668313f2b11c8eb2dbd1cf789ac29a4714179e65 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 3 Mar 2023 21:40:35 +0100 Subject: [PATCH 22/42] python: Fix boolean to nix conversion It didn't cause a problem because it converted True to 1, which still allowed 1 == True to succeed in Python (using assertEqual) --- python/src/python-to-nix.cc | 2 +- python/tests.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/python/src/python-to-nix.cc b/python/src/python-to-nix.cc index cc363abf5da..95ef3ed8ba5 100644 --- a/python/src/python-to-nix.cc +++ b/python/src/python-to-nix.cc @@ -73,7 +73,7 @@ nix::Value * pythonToNixValue(nix::EvalState & state, PyObject * obj) { auto v = state.allocValue(); - if (obj == Py_True && obj == Py_False) { + if (obj == Py_True || obj == Py_False) { v->mkBool(obj == Py_True); } else if (obj == Py_None) { v->mkNull(); diff --git a/python/tests.py b/python/tests.py index 3bb272aa1f5..0c21c53f127 100644 --- a/python/tests.py +++ b/python/tests.py @@ -66,5 +66,9 @@ def test_throw_null(self): except nix.ThrownNixError as e: self.assertEqual(e.args[0], "hello") + def test_booleans(self): + self.assertIs(nix.eval("assert a == true; a", vars=dict(a=True)), True) + self.assertIs(nix.eval("assert a == false; a", vars=dict(a=False)), False) + if __name__ == '__main__': unittest.main() From 96621a0c29d88c8619041af8798493fb7bb6fa25 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 3 Mar 2023 21:41:49 +0100 Subject: [PATCH 23/42] python: Add test for Null conversion --- python/tests.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/tests.py b/python/tests.py index 0c21c53f127..a59c06650a1 100644 --- a/python/tests.py +++ b/python/tests.py @@ -70,5 +70,8 @@ def test_booleans(self): self.assertIs(nix.eval("assert a == true; a", vars=dict(a=True)), True) self.assertIs(nix.eval("assert a == false; a", vars=dict(a=False)), False) + def test_null(self): + self.assertIs(nix.eval("assert a == null; a", vars=dict(a=None)), None) + if __name__ == '__main__': unittest.main() From 0503a739c6a1aea0e46a5031246df65740b47751 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 3 Mar 2023 21:52:44 +0100 Subject: [PATCH 24/42] python: Use assertRaises for tests --- python/tests.py | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/python/tests.py b/python/tests.py index a59c06650a1..a152dc34ea1 100644 --- a/python/tests.py +++ b/python/tests.py @@ -29,30 +29,22 @@ def test_ifd(self): def test_throw(self): errorString = "hello hi there\ntest" - try: + with self.assertRaises(nix.ThrownNixError) as cm: nix.eval("throw str", vars=dict(str=errorString)) - except nix.ThrownNixError as e: - self.assertEqual(e.args[0], errorString) + self.assertEquals(cm.exception.args[0], errorString) def test_syntax_error(self): - try: + with self.assertRaises(nix.NixError) as cm: nix.eval("{") - except nix.ThrownNixError as e: - self.assertTrue(False) - except nix.NixError as e: - self.assertTrue(True) def test_GIL_case(self): - try: + with self.assertRaises(nix.ThrownNixError) as cm: nix.eval("{ a = throw \"nope\"; }") - except nix.NixError as e: - self.assertEqual(e.args[0], "nope") + self.assertEqual(cm.exception.args[0], "nope") def test_infinity(self): - try: + with self.assertRaises(nix.NixError): nix.eval("let x = { inherit x; }; in x") - except nix.NixError as e: - self.assertTrue(True) def test_null_expression(self): # Null characters should be allowed in expressions, even if they aren't @@ -61,10 +53,9 @@ def test_null_expression(self): self.assertEqual(nix.eval("\"ab\x00cd\""), "ab") def test_throw_null(self): - try: + with self.assertRaises(nix.ThrownNixError) as cm: nix.eval("throw \"hello\x00there\"") - except nix.ThrownNixError as e: - self.assertEqual(e.args[0], "hello") + self.assertEqual(cm.exception.args[0], "hello") def test_booleans(self): self.assertIs(nix.eval("assert a == true; a", vars=dict(a=True)), True) From 1b0960fd53028f27eff6651c3593e56460416523 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Thu, 9 Mar 2023 20:10:07 +0100 Subject: [PATCH 25/42] python: We don't need to install the bindings into a subdirectly That in fact doesn't work for importing --- python/src/meson.build | 1 - 1 file changed, 1 deletion(-) diff --git a/python/src/meson.build b/python/src/meson.build index 1caf34b0a90..e95c1482d67 100644 --- a/python/src/meson.build +++ b/python/src/meson.build @@ -8,7 +8,6 @@ src = [ nix_bindings = py_installation.extension_module('nix', src, dependencies : [nix_expr_dep, nix_main_dep], install: true, - subdir: 'nix', cpp_args: [ '-std=c++17', # -Wnon-virtual-dtor is unnecessarily turned on by Meson From d8ce01be9d666760e331bd4b335bde75828e9645 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Thu, 9 Mar 2023 21:00:03 +0100 Subject: [PATCH 26/42] Remove python from manual build again Using a symlink hacky and breaks in some ways --- doc/manual/local.mk | 3 +-- doc/manual/src/SUMMARY.md.in | 3 --- doc/manual/src/python | 1 - 3 files changed, 1 insertion(+), 6 deletions(-) delete mode 120000 doc/manual/src/python diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 808d9267c85..63e7e61e437 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -2,8 +2,7 @@ ifeq ($(doc_generate),yes) MANUAL_SRCS := \ $(call rwildcard, $(d)/src, *.md) \ - $(call rwildcard, $(d)/src, */*.md) \ - $(call rwildcard, $(d)/../../python/doc, *.md) + $(call rwildcard, $(d)/src, */*.md) man-pages := $(foreach n, \ nix-env.1 nix-store.1 \ diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 2c44741068d..f783d590839 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -92,9 +92,6 @@ - [Files](command-ref/files.md) - [nix.conf](command-ref/conf-file.md) - [Architecture](architecture/architecture.md) -- [Python Bindings](python/index.md) - - [API](python/api.md) - - [Hacking](python/hacking.md) - [Glossary](glossary.md) - [Contributing](contributing/contributing.md) - [Hacking](contributing/hacking.md) diff --git a/doc/manual/src/python b/doc/manual/src/python deleted file mode 120000 index 14960743609..00000000000 --- a/doc/manual/src/python +++ /dev/null @@ -1 +0,0 @@ -../../../python/doc \ No newline at end of file From 9627862ed9fe46544160724d3006eba314af8d2e Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Thu, 9 Mar 2023 21:00:30 +0100 Subject: [PATCH 27/42] Very simple Nix source filtering So that we don't have to rebuild Nix every time the python bindings change --- flake.nix | 2 +- python/examples/buildPythonApplication/default.nix | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 9af342b1c88..5bc21016892 100644 --- a/flake.nix +++ b/flake.nix @@ -331,7 +331,7 @@ name = "nix-${version}"; inherit version; - src = sourceByRegexInverted [ "tests/nixos/.*" "tests/installer/.*" ] self; + src = sourceByRegexInverted [ "tests/nixos/.*" "tests/installer/.*" "python/*" "flake.nix" ] self; VERSION_SUFFIX = versionSuffix; outputs = [ "out" "dev" "doc" ]; diff --git a/python/examples/buildPythonApplication/default.nix b/python/examples/buildPythonApplication/default.nix index 88c9c390e30..7d05140df89 100644 --- a/python/examples/buildPythonApplication/default.nix +++ b/python/examples/buildPythonApplication/default.nix @@ -1,6 +1,6 @@ { nixpkgsSrc ? fetchTarball "https://github.com/NixOS/nixpkgs/archive/545c7a31e5dedea4a6d372712a18e00ce097d462.tar.gz", - nixSrc ? fetchTarball "https://github.com/tweag/nix/archive/9b49f6902c5511e96a2a822bc36f245caf78de3f.tar.gz", + nixSrc ? ../../.., }: let pkgs = import nixpkgsSrc { overlays = []; config = {}; }; nix = (import nixSrc).default; From a91f312e2630aaaf23d5ff7d2d7a0fde8b7d4900 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Thu, 9 Mar 2023 21:01:05 +0100 Subject: [PATCH 28/42] Fix the buildPythonApplication example --- python/examples/buildPythonApplication/hello/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/examples/buildPythonApplication/hello/__init__.py b/python/examples/buildPythonApplication/hello/__init__.py index 786e4327b31..710a150d266 100644 --- a/python/examples/buildPythonApplication/hello/__init__.py +++ b/python/examples/buildPythonApplication/hello/__init__.py @@ -1,4 +1,4 @@ import nix def greet(): - print("Evaluating 1 + 1 in Nix gives:" + nix.eval("1 + 1")) + print("Evaluating 1 + 1 in Nix gives:" + str(nix.eval("1 + 1"))) From 06d713509b9071db7bd716f52f902ef850c2c1c5 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Thu, 9 Mar 2023 23:31:27 +0100 Subject: [PATCH 29/42] Fix some unset variable problems with vars-and-functions.sh - BASH_SOURCE[0] doesn't exist for evaluations under `eval` - NIX_STORE may not be set --- tests/common/vars-and-functions.sh.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/common/vars-and-functions.sh.in b/tests/common/vars-and-functions.sh.in index a9e6c802fe5..2ae12cc6b13 100644 --- a/tests/common/vars-and-functions.sh.in +++ b/tests/common/vars-and-functions.sh.in @@ -4,7 +4,7 @@ if [[ -z "${COMMON_VARS_AND_FUNCTIONS_SH_SOURCED-}" ]]; then COMMON_VARS_AND_FUNCTIONS_SH_SOURCED=1 -export PS4='+(${BASH_SOURCE[0]}:$LINENO) ' +export PS4='+(${BASH_SOURCE[0]:-}:$LINENO) ' export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/${TEST_NAME:-default} export NIX_STORE_DIR @@ -20,7 +20,7 @@ export NIX_CONF_DIR=$TEST_ROOT/etc export NIX_DAEMON_SOCKET_PATH=$TEST_ROOT/dSocket unset NIX_USER_CONF_FILES export _NIX_TEST_SHARED=$TEST_ROOT/shared -if [[ -n $NIX_STORE ]]; then +if [[ -n ${NIX_STORE:-} ]]; then export _NIX_TEST_NO_SANDBOX=1 fi export _NIX_IN_TEST=$TEST_ROOT/shared From 34c8519cf3b478e2f2009e297d81b650caff89d6 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 10 Mar 2023 00:09:16 +0100 Subject: [PATCH 30/42] assertEquals -> assertEqual The former is deprecated --- python/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/tests.py b/python/tests.py index a152dc34ea1..131d3180522 100644 --- a/python/tests.py +++ b/python/tests.py @@ -31,7 +31,7 @@ def test_throw(self): errorString = "hello hi there\ntest" with self.assertRaises(nix.ThrownNixError) as cm: nix.eval("throw str", vars=dict(str=errorString)) - self.assertEquals(cm.exception.args[0], errorString) + self.assertEqual(cm.exception.args[0], errorString) def test_syntax_error(self): with self.assertRaises(nix.NixError) as cm: From 91b0740ac2041a61b0c6f3aff6c9f12f5db9c5d4 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 10 Mar 2023 00:33:11 +0100 Subject: [PATCH 31/42] Alternate approach to calling init.sh Previously the python bindings derivation would use Nix as its source, call configure on that, just to extract the two files tests/init.sh and (the generated one) tests/common/vars-and-functions.sh The new approach is to instead have a separate derivation extracting these two files and having a small wrapper around them --- python/default.nix | 75 +++++++++++++++++++++++++++++++--------------- python/meson.build | 6 +--- python/test.sh | 4 +-- 3 files changed, 53 insertions(+), 32 deletions(-) diff --git a/python/default.nix b/python/default.nix index a05726b6f0d..5c4e0fdb78a 100644 --- a/python/default.nix +++ b/python/default.nix @@ -1,4 +1,44 @@ -{ self, system, lib, python, clang-tools, ninja, meson, nix, mkShell }: +{ + self, + system, + lib, + python, + boost, + clang-tools, + pkg-config, + ninja, + meson, + nix, + mkShell, + recurseIntoAttrs, + isShell ? false, +}: +let + # Extracts tests/init.sh + testScripts = nix.overrideAttrs (old: { + name = "nix-test-scripts-${old.version}"; + outputs = [ "out" ]; + separateDebugInfo = false; + buildPhase = '' + make tests/{init.sh,common/vars-and-functions.sh} + ''; + script = '' + pushd ${placeholder "out"}/libexec >/dev/null + source init.sh + popd >/dev/null + ''; + passAsFile = [ "script" ]; + installPhase = '' + rm -rf "$out" + mkdir -p "$out"/{libexec/common,share/bash} + cp tests/init.sh "$out"/libexec + cp tests/common/vars-and-functions.sh "$out"/libexec/common + + cp "$scriptPath" "$out"/share/bash/nix-test.sh + ''; + dontFixup = true; + }); +in python.pkgs.buildPythonPackage { name = "nix"; format = "other"; @@ -7,33 +47,20 @@ python.pkgs.buildPythonPackage { strictDeps = true; - nativeBuildInputs = lib.optionals (nix != null) nix.nativeBuildInputs ++ [ + nativeBuildInputs = [ ninja + pkg-config (meson.override { python3 = python; }) - nix - ]; - - buildInputs = lib.optionals (nix != null) nix.buildInputs ++ [ - nix - ]; - - # We need to be able to generate tests/common.sh, which requires running - # `make`, which requires having run `autoreconf` and `./configure`. - # So we propagate `autoreconfHook` from nix.nativeBuildInputs for that to - # work, but after that we also need to cd into the python directory and run the - # meson configure phase for the python bindings. - # Can't use `postConfigure` for this because that would create a loop since - # `mesonConfigurePhase` calls `postConfigure` itself. - # A small problem with this is that `{pre,post}Configure` get run twice - dontUseMesonConfigure = true; - preBuild = '' - cd python - mesonConfigurePhase - ''; + ] ++ lib.optional (!isShell) nix; + + buildInputs = [ + boost + ] ++ lib.optional (!isShell) nix; mesonBuildType = "release"; doInstallCheck = true; + TEST_SCRIPTS = testScripts; installCheckPhase = "meson test -v"; passthru = { @@ -42,9 +69,9 @@ python.pkgs.buildPythonPackage { packages = [ clang-tools ]; + TEST_SCRIPTS = testScripts; inputsFrom = [ - self.devShells.${system}.default - (nix.python-bindings.override { nix = null; }) + (nix.python-bindings.override { isShell = true; }) ]; }; }; diff --git a/python/meson.build b/python/meson.build index e2f9294c02d..6a212749c3b 100644 --- a/python/meson.build +++ b/python/meson.build @@ -13,10 +13,6 @@ subdir('src') fs = import('fs') -nix_root = fs.parent(meson.project_source_root()) -run_command('make', '-C', nix_root, 'tests/common/vars-and-functions.sh', check: true) - env = environment() env.prepend('PYTHONPATH', fs.parent(nix_bindings.full_path())) -bash = find_program('bash') -test('python test', bash, args : files('test.sh'), env : env) +test('python test', find_program('bash'), args : files('test.sh'), env : env) diff --git a/python/test.sh b/python/test.sh index ca13ce5f6ea..3a1ad8ae250 100644 --- a/python/test.sh +++ b/python/test.sh @@ -1,7 +1,5 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -cd "$SCRIPT_DIR"/../tests - -source init.sh +source "$TEST_SCRIPTS"/share/bash/nix-test.sh python "$SCRIPT_DIR"/tests.py From 56ef3e361db6524b85dc94bb9514040418a43e62 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 10 Mar 2023 00:35:02 +0100 Subject: [PATCH 32/42] Don't depend on all files for the python bindings --- python/default.nix | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/python/default.nix b/python/default.nix index 5c4e0fdb78a..c28a825d6cb 100644 --- a/python/default.nix +++ b/python/default.nix @@ -43,7 +43,15 @@ python.pkgs.buildPythonPackage { name = "nix"; format = "other"; - src = self; + src = builtins.path { + path = ./.; + filter = path: type: + path == toString ./meson.build + || path == toString ./tests.py + || path == toString ./test.sh + || lib.hasPrefix (toString ./src) path; + }; + strictDeps = true; From 8d734cdca97b3ed9374fcca80b3c8fe9a9b3de91 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 10 Mar 2023 00:35:31 +0100 Subject: [PATCH 33/42] Make buildPythonApplication test work Tests that the Nix bindings can be used as a Python dependency in Nix builds --- flake.nix | 2 + python/default.nix | 5 ++ .../buildPythonApplication/default.nix | 47 ++++++++++++------- .../buildPythonApplication/hello/__init__.py | 2 +- .../examples/buildPythonApplication/setup.py | 1 - 5 files changed, 37 insertions(+), 20 deletions(-) diff --git a/flake.nix b/flake.nix index 5bc21016892..df2b15ea2c8 100644 --- a/flake.nix +++ b/flake.nix @@ -506,6 +506,8 @@ perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nix.perl-bindings); pythonBindings = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix.python-bindings); + # TODO: recurseIntoAttrs or combine multiple tests into a single one + pythonBindingsTests = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix.python-bindings.tests.example-buildPythonApplication); # Binary tarball for various platforms, containing a Nix store # with the closure of 'nix' package, and the second half of diff --git a/python/default.nix b/python/default.nix index c28a825d6cb..6ff0ce43825 100644 --- a/python/default.nix +++ b/python/default.nix @@ -73,6 +73,11 @@ python.pkgs.buildPythonPackage { passthru = { exampleEnv = python.withPackages (p: [ nix.python-bindings ]); + tests = { + example-buildPythonApplication = import ./examples/buildPythonApplication { + inherit nix system testScripts; + }; + }; shell = mkShell { packages = [ clang-tools diff --git a/python/examples/buildPythonApplication/default.nix b/python/examples/buildPythonApplication/default.nix index 7d05140df89..96fac6b9b19 100644 --- a/python/examples/buildPythonApplication/default.nix +++ b/python/examples/buildPythonApplication/default.nix @@ -1,26 +1,37 @@ { - nixpkgsSrc ? fetchTarball "https://github.com/NixOS/nixpkgs/archive/545c7a31e5dedea4a6d372712a18e00ce097d462.tar.gz", - nixSrc ? ../../.., -}: let - pkgs = import nixpkgsSrc { overlays = []; config = {}; }; - nix = (import nixSrc).default; + system ? builtins.currentSystem, + pkgs ? import (fetchTarball { + url = "https://github.com/NixOS/nixpkgs/archive/545c7a31e5dedea4a6d372712a18e00ce097d462.tar.gz"; + sha256 = "1dbsi2ccq8x0hyl8n0hisigj8q19amvj9irzfbgy4b3szb6x2y6l"; + }) { + config = {}; + overlays = []; + inherit system; + }, + nix ? (import ../../..).default, + testScripts, +}: +let python = pkgs.python3; nixBindings = nix.python-bindings.override { inherit python; }; in python.pkgs.buildPythonApplication { pname = "hello-nix"; version = "0.1"; - src = ./.; - propagatedBuildInputs = [ - nixBindings - #python.pkgs.requests + src = builtins.path { + path = ./.; + filter = path: type: + pkgs.lib.hasPrefix (toString ./hello) path + || path == toString ./setup.py; + }; + propagatedBuildInputs = [ nixBindings ]; + doInstallCheck = true; + nativeCheckInputs = [ + nix ]; - #preBuild = '' - # echo $PYTHONPATH - # exit 1 - #''; - dontUseSetuptoolsShellHook = true; - passthru.nixBindings = nixBindings; - #checkPhase = '' - # ls -laa $out/hello-nix - #''; + installCheckPhase = '' + ( + source ${testScripts}/share/bash/nix-test.sh + $out/bin/hello-nix + ) + ''; } diff --git a/python/examples/buildPythonApplication/hello/__init__.py b/python/examples/buildPythonApplication/hello/__init__.py index 710a150d266..af1e3a19a75 100644 --- a/python/examples/buildPythonApplication/hello/__init__.py +++ b/python/examples/buildPythonApplication/hello/__init__.py @@ -1,4 +1,4 @@ import nix def greet(): - print("Evaluating 1 + 1 in Nix gives:" + str(nix.eval("1 + 1"))) + print("Evaluating 1 + 1 in Nix gives: " + str(nix.eval("1 + 1"))) diff --git a/python/examples/buildPythonApplication/setup.py b/python/examples/buildPythonApplication/setup.py index 3fde1535e9b..c49b416d4e1 100644 --- a/python/examples/buildPythonApplication/setup.py +++ b/python/examples/buildPythonApplication/setup.py @@ -9,5 +9,4 @@ 'hello-nix = hello:greet', ], }, - #install_requires=['nix'], ) From f5fd4358b631beeff9854e7fa4cd95bb6a8c642e Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 10 Mar 2023 18:33:33 +0100 Subject: [PATCH 34/42] Fix dev shell --- python/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/default.nix b/python/default.nix index 6ff0ce43825..077642396e3 100644 --- a/python/default.nix +++ b/python/default.nix @@ -61,7 +61,7 @@ python.pkgs.buildPythonPackage { (meson.override { python3 = python; }) ] ++ lib.optional (!isShell) nix; - buildInputs = [ + buildInputs = nix.propagatedBuildInputs ++ [ boost ] ++ lib.optional (!isShell) nix; From 68996d8b1edb3026f5188f7ced6b9e02f1a044d7 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 10 Mar 2023 19:20:30 +0100 Subject: [PATCH 35/42] Add debugging capability to dev shell Docs still needed --- python/default.nix | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/python/default.nix b/python/default.nix index 077642396e3..6eeed48128d 100644 --- a/python/default.nix +++ b/python/default.nix @@ -4,16 +4,22 @@ lib, python, boost, + gdb, clang-tools, pkg-config, ninja, meson, nix, mkShell, + enableDebugging, recurseIntoAttrs, isShell ? false, }: let + _python = python; +in +let + python = _python.override { self = enableDebugging _python; }; # Extracts tests/init.sh testScripts = nix.overrideAttrs (old: { name = "nix-test-scripts-${old.version}"; @@ -81,6 +87,7 @@ python.pkgs.buildPythonPackage { shell = mkShell { packages = [ clang-tools + gdb ]; TEST_SCRIPTS = testScripts; inputsFrom = [ From 22c9e48724f4be2d9b473984e191834aa9fc8cf1 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 10 Mar 2023 19:22:35 +0100 Subject: [PATCH 36/42] Convert from eval to callExprString API --- python/doc/index.md | 2 +- .../buildPythonApplication/hello/__init__.py | 2 +- python/src/eval.cc | 24 +++++++------- python/src/internal/python-to-nix.hh | 1 - python/src/python-module.cc | 2 +- python/src/python-to-nix.cc | 32 ------------------- python/tests.py | 29 +++++++++-------- 7 files changed, 31 insertions(+), 61 deletions(-) diff --git a/python/doc/index.md b/python/doc/index.md index 53b29fff242..3db5cb42436 100644 --- a/python/doc/index.md +++ b/python/doc/index.md @@ -11,7 +11,7 @@ $ nix run github:NixOS/nix#nix.python-bindings.exampleEnv Python 3.10.8 (main, Oct 11 2022, 11:35:05) [GCC 11.3.0] on linux Type "help", "copyright", "credits" or "license" for more information. >>> import nix ->>> nix.eval('"Hello ${name}!"', vars=dict(name="Python")) +>>> nix.callExprString('"Hello ${name}!"', arg={"name": "Python"})) 'Hello Python!' ``` diff --git a/python/examples/buildPythonApplication/hello/__init__.py b/python/examples/buildPythonApplication/hello/__init__.py index af1e3a19a75..179053c3f7a 100644 --- a/python/examples/buildPythonApplication/hello/__init__.py +++ b/python/examples/buildPythonApplication/hello/__init__.py @@ -1,4 +1,4 @@ import nix def greet(): - print("Evaluating 1 + 1 in Nix gives: " + str(nix.eval("1 + 1"))) + print("Evaluating 1 + 1 in Nix gives: " + str(nix.callExprString("_: 1 + 1", None))) diff --git a/python/src/eval.cc b/python/src/eval.cc index 9ad3b44b910..b7ddc4ed454 100644 --- a/python/src/eval.cc +++ b/python/src/eval.cc @@ -17,17 +17,16 @@ const char * currentExceptionTypeName() return res ? res : "(null)"; } -static PyObject * _eval(const std::string & expression, PyObject * vars) +static PyObject * _eval(const std::string & expression, PyObject * argument) { nix::Strings storePath; nix::EvalState state(storePath, nix::openStore()); - nix::Env * env = nullptr; - auto staticEnv = pythonToNixEnv(state, vars, &env); - if (!staticEnv) { + auto nixArgument = pythonToNixValue(state, argument); + if (!nixArgument) { return nullptr; } - auto staticEnvPointer = std::make_shared(*staticEnv); + nix::Value fun; nix::Value v; @@ -44,8 +43,10 @@ static PyObject * _eval(const std::string & expression, PyObject * vars) }); // TODO: Should the "." be something else here? - auto e = state.parseExprFromString(expression, ".", staticEnvPointer); - e->eval(state, *env, v); + auto e = state.parseExprFromString(expression, "."); + state.eval(e, fun); + // TODO: Add position + state.callFunction(fun, *nixArgument, v, noPos); state.forceValueDeep(v); } @@ -53,16 +54,17 @@ static PyObject * _eval(const std::string & expression, PyObject * vars) return nixToPythonObject(state, v, context); } +// TODO: Rename this function to callExprString, matching the Python name PyObject * eval(PyObject * self, PyObject * args, PyObject * keywds) { PyObject * expressionObject; - PyObject * vars = nullptr; + PyObject * argument = nullptr; - const char * kwlist[] = {"expression", "vars", nullptr}; + const char * kwlist[] = {"expression", "arg", nullptr}; // See https://docs.python.org/3/c-api/arg.html for the magic string if (!PyArg_ParseTupleAndKeywords( - args, keywds, "U|O!", const_cast(kwlist), &expressionObject, &PyDict_Type, &vars)) { + args, keywds, "UO", const_cast(kwlist), &expressionObject, &argument)) { return nullptr; } @@ -75,7 +77,7 @@ PyObject * eval(PyObject * self, PyObject * args, PyObject * keywds) std::string expression(expressionBase, expressionSize); try { - return _eval(expression, vars); + return _eval(expression, argument); } catch (nix::ThrownError & e) { return PyErr_Format(ThrownNixError, "%s", e.message().c_str()); } catch (nix::Error & e) { diff --git a/python/src/internal/python-to-nix.hh b/python/src/internal/python-to-nix.hh index e922323893b..a7dde2195b6 100644 --- a/python/src/internal/python-to-nix.hh +++ b/python/src/internal/python-to-nix.hh @@ -9,5 +9,4 @@ namespace nix::python { nix::Value * pythonToNixValue(nix::EvalState & state, PyObject * obj); -std::optional pythonToNixEnv(nix::EvalState & state, PyObject * vars, nix::Env ** env); } // namespace nix::python diff --git a/python/src/python-module.cc b/python/src/python-module.cc index 3c297550a26..e9a9ffba8b6 100644 --- a/python/src/python-module.cc +++ b/python/src/python-module.cc @@ -15,7 +15,7 @@ PyObject * ThrownNixError = nullptr; PyObject * NixError = nullptr; static PyMethodDef NixMethods[] = { - {"eval", (PyCFunction) eval, METH_VARARGS | METH_KEYWORDS, "Eval nix expression"}, {NULL, NULL, 0, NULL}}; + {"callExprString", (PyCFunction) eval, METH_VARARGS | METH_KEYWORDS, "Eval nix expression"}, {NULL, NULL, 0, NULL}}; static struct PyModuleDef nixmodule = { PyModuleDef_HEAD_INIT, "nix", "Nix expression bindings", diff --git a/python/src/python-to-nix.cc b/python/src/python-to-nix.cc index 95ef3ed8ba5..eb836fd59cd 100644 --- a/python/src/python-to-nix.cc +++ b/python/src/python-to-nix.cc @@ -130,36 +130,4 @@ nix::Value * pythonToNixValue(nix::EvalState & state, PyObject * obj) } return v; } - -std::optional pythonToNixEnv(nix::EvalState & state, PyObject * vars, nix::Env ** env) -{ - Py_ssize_t pos = 0; - PyObject *key = nullptr, *val = nullptr; - - *env = &state.allocEnv(vars ? PyDict_Size(vars) : 0); - (*env)->up = &state.baseEnv; - - nix::StaticEnv staticEnv(false, state.staticBaseEnv.get()); - - if (!vars) { - return staticEnv; - } - - auto displ = 0; - while (PyDict_Next(vars, &pos, &key, &val)) { - auto name = checkAttrKey(key); - if (!name) { - return {}; - } - - auto attrVal = pythonToNixValue(state, val); - if (!attrVal) { - return {}; - } - staticEnv.vars.emplace_back(state.symbols.create(name), displ); - (*env)->values[displ++] = attrVal; - } - - return staticEnv; -} } // namespace nix::python diff --git a/python/tests.py b/python/tests.py index 131d3180522..f84e8a0f3dd 100644 --- a/python/tests.py +++ b/python/tests.py @@ -5,19 +5,20 @@ class TestPythonNix(unittest.TestCase): def test_dict(self): val = dict(a=1) - self.assertEqual(nix.eval("a", vars=dict(a=val)), val) + self.assertEqual(nix.callExprString("{ a }: a", arg=dict(a=val)), val) def test_string(self): - self.assertEqual(nix.eval("a", vars=dict(a="foo")), "foo") + self.assertEqual(nix.callExprString("{ a }: a", arg=dict(a="foo")), "foo") def test_bool(self): - self.assertEqual(nix.eval("a", vars=dict(a=True)), True) + self.assertEqual(nix.callExprString("{ a }: a", arg=dict(a=True)), True) def test_none(self): - self.assertEqual(nix.eval("a", vars=dict(a=None)), None) + self.assertEqual(nix.callExprString("{ a }: a", arg=dict(a=None)), None) def test_ifd(self): expression = """ + {}: builtins.readFile (derivation { name = "test"; args = [ "-c" "printf \\"%s\\" test > $out" ]; @@ -25,44 +26,44 @@ def test_ifd(self): system = builtins.currentSystem; }) """ - self.assertEqual(nix.eval(expression, vars=dict()), "test") + self.assertEqual(nix.callExprString(expression, arg={}), "test") def test_throw(self): errorString = "hello hi there\ntest" with self.assertRaises(nix.ThrownNixError) as cm: - nix.eval("throw str", vars=dict(str=errorString)) + nix.callExprString("{ str }: throw str", arg=dict(str=errorString)) self.assertEqual(cm.exception.args[0], errorString) def test_syntax_error(self): with self.assertRaises(nix.NixError) as cm: - nix.eval("{") + nix.callExprString("{", arg={}) def test_GIL_case(self): with self.assertRaises(nix.ThrownNixError) as cm: - nix.eval("{ a = throw \"nope\"; }") + nix.callExprString("{}: { a = throw \"nope\"; }", arg={}) self.assertEqual(cm.exception.args[0], "nope") def test_infinity(self): with self.assertRaises(nix.NixError): - nix.eval("let x = { inherit x; }; in x") + nix.callExprString("{}: let x = { inherit x; }; in x", arg={}) def test_null_expression(self): # Null characters should be allowed in expressions, even if they aren't # very useful really, though at least null's should be supported in # strings in the future https://github.com/NixOS/nix/issues/1307) - self.assertEqual(nix.eval("\"ab\x00cd\""), "ab") + self.assertEqual(nix.callExprString("{}: \"ab\x00cd\"", arg={}), "ab") def test_throw_null(self): with self.assertRaises(nix.ThrownNixError) as cm: - nix.eval("throw \"hello\x00there\"") + nix.callExprString("{}: throw \"hello\x00there\"", arg={}) self.assertEqual(cm.exception.args[0], "hello") def test_booleans(self): - self.assertIs(nix.eval("assert a == true; a", vars=dict(a=True)), True) - self.assertIs(nix.eval("assert a == false; a", vars=dict(a=False)), False) + self.assertIs(nix.callExprString("{ a }: assert a == true; a", arg=dict(a=True)), True) + self.assertIs(nix.callExprString("{ a }: assert a == false; a", arg=dict(a=False)), False) def test_null(self): - self.assertIs(nix.eval("assert a == null; a", vars=dict(a=None)), None) + self.assertIs(nix.callExprString("{ a }: assert a == null; a", arg=dict(a=None)), None) if __name__ == '__main__': unittest.main() From fb5884e8246ff30950a8b919f036722744a384a2 Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Fri, 14 Apr 2023 13:48:22 +0200 Subject: [PATCH 37/42] python api: write API docs for callExprString --- python/doc/api.md | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/python/doc/api.md b/python/doc/api.md index e586330cd13..116ed60dc30 100644 --- a/python/doc/api.md +++ b/python/doc/api.md @@ -1,3 +1,18 @@ -# Experimental Python Bindings API +# Experimental Python Bindings + +## callExprString + +```python +nix.callExprString(expression: str, arg) +``` +Parse a nix expression, then call it as a nix function. + +Note that this function is experimental and subject to change based on known issues and feedback. + +**Parameters:**, + `expression` (str): The string containing a nix expression. + `arg`: the argument to pass to the function + +**Returns:** + `result`: the result of the function invocation, converted to python datatypes. -This is the API! From f1442f8531295de3c8ac591d47d13746b7d3c4e2 Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Fri, 14 Apr 2023 13:51:07 +0200 Subject: [PATCH 38/42] PyInit_Nix: elaborate on build hook workaround --- python/src/python-module.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/python/src/python-module.cc b/python/src/python-module.cc index e9a9ffba8b6..c1b1eb51e83 100644 --- a/python/src/python-module.cc +++ b/python/src/python-module.cc @@ -25,6 +25,7 @@ static struct PyModuleDef nixmodule = { extern "C" _public_ PyObject * PyInit_nix(void) { + // These bindings can trigger builds via IFD. This means we need the build-hook to work. // By default, Nix sets the build-hook to be "$(readlink /proc/self/exe) __build-remote", expecting the current // binary to be Nix itself. But when we call the Nix library from Python this isn't the case, the current binary is // Python then So we need to change this default, pointing it to the Nix binary instead From 0d52ddc11cbd4c25b91ba23cb34e12c0a9ef912d Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Fri, 14 Apr 2023 14:01:05 +0200 Subject: [PATCH 39/42] buildPythonApplication: fix tests --- python/default.nix | 2 +- python/examples/buildPythonApplication/default.nix | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/default.nix b/python/default.nix index 6eeed48128d..43410878636 100644 --- a/python/default.nix +++ b/python/default.nix @@ -81,7 +81,7 @@ python.pkgs.buildPythonPackage { exampleEnv = python.withPackages (p: [ nix.python-bindings ]); tests = { example-buildPythonApplication = import ./examples/buildPythonApplication { - inherit nix system testScripts; + inherit nix system testScripts python; }; }; shell = mkShell { diff --git a/python/examples/buildPythonApplication/default.nix b/python/examples/buildPythonApplication/default.nix index 96fac6b9b19..21194be5c3c 100644 --- a/python/examples/buildPythonApplication/default.nix +++ b/python/examples/buildPythonApplication/default.nix @@ -8,11 +8,11 @@ overlays = []; inherit system; }, + python ? pkgs.python3, nix ? (import ../../..).default, testScripts, }: let - python = pkgs.python3; nixBindings = nix.python-bindings.override { inherit python; }; in python.pkgs.buildPythonApplication { pname = "hello-nix"; From 51440a3539edd5ca9bdf8ee75f9cba6f3b9dedc2 Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Fri, 14 Apr 2023 14:07:24 +0200 Subject: [PATCH 40/42] python-bindings: use prev.callPackage to pass args --- flake.nix | 5 +---- python/default.nix | 15 +++++++-------- .../examples/buildPythonApplication/default.nix | 6 +++--- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/flake.nix b/flake.nix index df2b15ea2c8..635dce2192b 100644 --- a/flake.nix +++ b/flake.nix @@ -439,10 +439,7 @@ postUnpack = "sourceRoot=$sourceRoot/perl"; }); - passthru.python-bindings = final.callPackage ./python { - inherit self system; - python = final.python3; - }; + passthru.python-bindings = prev.callPackage ./python { }; meta.platforms = lib.platforms.unix; }); diff --git a/python/default.nix b/python/default.nix index 43410878636..9bec351d950 100644 --- a/python/default.nix +++ b/python/default.nix @@ -1,8 +1,7 @@ { - self, system, lib, - python, + python3, boost, gdb, clang-tools, @@ -16,10 +15,10 @@ isShell ? false, }: let - _python = python; + _python = python3; in let - python = _python.override { self = enableDebugging _python; }; + python3 = _python.override { self = enableDebugging _python; }; # Extracts tests/init.sh testScripts = nix.overrideAttrs (old: { name = "nix-test-scripts-${old.version}"; @@ -45,7 +44,7 @@ let dontFixup = true; }); in -python.pkgs.buildPythonPackage { +python3.pkgs.buildPythonPackage { name = "nix"; format = "other"; @@ -64,7 +63,7 @@ python.pkgs.buildPythonPackage { nativeBuildInputs = [ ninja pkg-config - (meson.override { python3 = python; }) + (meson.override { inherit python3; }) ] ++ lib.optional (!isShell) nix; buildInputs = nix.propagatedBuildInputs ++ [ @@ -78,10 +77,10 @@ python.pkgs.buildPythonPackage { installCheckPhase = "meson test -v"; passthru = { - exampleEnv = python.withPackages (p: [ nix.python-bindings ]); + exampleEnv = python3.withPackages (p: [ nix.python-bindings ]); tests = { example-buildPythonApplication = import ./examples/buildPythonApplication { - inherit nix system testScripts python; + inherit nix system testScripts python3; }; }; shell = mkShell { diff --git a/python/examples/buildPythonApplication/default.nix b/python/examples/buildPythonApplication/default.nix index 21194be5c3c..9bf26a17b63 100644 --- a/python/examples/buildPythonApplication/default.nix +++ b/python/examples/buildPythonApplication/default.nix @@ -8,13 +8,13 @@ overlays = []; inherit system; }, - python ? pkgs.python3, + python3 ? pkgs.python3, nix ? (import ../../..).default, testScripts, }: let - nixBindings = nix.python-bindings.override { inherit python; }; -in python.pkgs.buildPythonApplication { + nixBindings = nix.python-bindings.override { inherit python3; }; +in python3.pkgs.buildPythonApplication { pname = "hello-nix"; version = "0.1"; src = builtins.path { From 2cb9dd0d217422293e402676b95c7a6b0ca8b639 Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Fri, 14 Apr 2023 15:42:41 +0200 Subject: [PATCH 41/42] python-to-nix: don't forward declare PyObject, doesn't work everywhere --- python/src/internal/python-to-nix.hh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/src/internal/python-to-nix.hh b/python/src/internal/python-to-nix.hh index a7dde2195b6..a4853fb0d8c 100644 --- a/python/src/internal/python-to-nix.hh +++ b/python/src/internal/python-to-nix.hh @@ -1,7 +1,6 @@ #pragma once -struct PyObject; - +#include #include #include From 57e71a381d6b72165abcad8210c6536cebd947f3 Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Wed, 19 Apr 2023 14:58:03 +0200 Subject: [PATCH 42/42] python: initNix -> initLibStore --- python/meson.build | 1 - python/src/meson.build | 2 +- python/src/python-module.cc | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/python/meson.build b/python/meson.build index 6a212749c3b..eb31ffca1ba 100644 --- a/python/meson.build +++ b/python/meson.build @@ -7,7 +7,6 @@ python_mod = import('python') py_installation = python_mod.find_installation() nix_expr_dep = dependency('nix-expr', required: true) -nix_main_dep = dependency('nix-main', required: true) subdir('src') diff --git a/python/src/meson.build b/python/src/meson.build index e95c1482d67..49b94a2b752 100644 --- a/python/src/meson.build +++ b/python/src/meson.build @@ -6,7 +6,7 @@ src = [ ] nix_bindings = py_installation.extension_module('nix', src, - dependencies : [nix_expr_dep, nix_main_dep], + dependencies : [nix_expr_dep], install: true, cpp_args: [ '-std=c++17', diff --git a/python/src/python-module.cc b/python/src/python-module.cc index c1b1eb51e83..67549577dc3 100644 --- a/python/src/python-module.cc +++ b/python/src/python-module.cc @@ -32,7 +32,7 @@ extern "C" _public_ PyObject * PyInit_nix(void) nix::settings.buildHook = nix::settings.nixBinDir + "/nix __build-remote"; // And by setting buildHook before calling initNix, we can override the defaults without overriding the // user-provided options from the config files - nix::initNix(); + nix::initLibStore(); nix::initGC(); PyObjPtr m(PyModule_Create(&nixmodule));