Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Tracy-based expr evaluation profiler #9967

Draft
wants to merge 5 commits into
base: master
Choose a base branch
from
+222 −4
Draft

Tracy-based expr evaluation profiler #9967

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
0905dea
Naive build system setup
picnoir Feb 6, 2024
f2e832d
Trace the eval.cc functions with Tracy
picnoir Feb 6, 2024
b35cc13
Tracing EvalState members
picnoir Feb 7, 2024
376d9dd
Tracing primops
picnoir Feb 7, 2024
3ea4870
Tracy: autotools setup
picnoir Feb 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Trace the eval.cc functions with Tracy 
We're leveraging the ZoneTransientN macro to send dynamic strings
containing the expression type, file and position in that file of the
expression Nix is currently evaluating.

We had to add a new showExprType method to the Expr class to get a
const string containing the name of an expression.
  • Loading branch information
@picnoir
picnoir committed Feb 8, 2024
commit f2e832d0f81e392f607cb95c2c3e2d3cf7f9d8d1
32 changes: 31 additions & 1 deletion src/libexpr/eval.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,11 @@

#endif

#define TRACY_TRACE(evalstate, expr) \
std::ostringstream tracyss; \
tracyss << evalstate.positions[(expr)->getPos()] << " " << (expr)->showExprType(); \
ZoneTransientN(nix, tracyss.str().c_str(), true);

using json = nlohmann::json;

namespace nix {
Expand Down Expand Up @@ -1204,29 +1209,34 @@ void Expr::eval(EvalState & state, Env & env, Value & v)

void ExprInt::eval(EvalState & state, Env & env, Value & v)
{
TRACY_TRACE(state, this);
v = this->v;
}


void ExprFloat::eval(EvalState & state, Env & env, Value & v)
{
TRACY_TRACE(state, this);
v = this->v;
}

void ExprString::eval(EvalState & state, Env & env, Value & v)
{
TRACY_TRACE(state, this);
v = this->v;
}


void ExprPath::eval(EvalState & state, Env & env, Value & v)
{
TRACY_TRACE(state, this);
v = this->v;
}


void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
{
TRACY_TRACE(state, this);
v.mkAttrs(state.buildBindings(attrs.size() + dynamicAttrs.size()).finish());
auto dynamicEnv = &env;

Expand Down Expand Up @@ -1311,6 +1321,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)

void ExprLet::eval(EvalState & state, Env & env, Value & v)
{
TRACY_TRACE(state, this);
/* Create a new environment that contains the attributes in this
`let'. */
Env & env2(state.allocEnv(attrs->attrs.size()));
Expand All @@ -1329,6 +1340,7 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v)

void ExprList::eval(EvalState & state, Env & env, Value & v)
{
TRACY_TRACE(state, this);
state.mkList(v, elems.size());
for (auto [n, v2] : enumerate(v.listItems()))
const_cast<Value * &>(v2) = elems[n]->maybeThunk(state, env);
Expand All @@ -1337,6 +1349,7 @@ void ExprList::eval(EvalState & state, Env & env, Value & v)

Value * ExprList::maybeThunk(EvalState & state, Env & env)
{
TRACY_TRACE(state, this);
if (elems.empty()) {
return &state.vEmptyList;
}
Expand All @@ -1346,6 +1359,7 @@ Value * ExprList::maybeThunk(EvalState & state, Env & env)

void ExprVar::eval(EvalState & state, Env & env, Value & v)
{
TRACY_TRACE(state, this);
Value * v2 = state.lookupVar(&env, *this, false);
state.forceValue(*v2, pos);
v = *v2;
Expand Down Expand Up @@ -1373,7 +1387,7 @@ static std::string showAttrPath(EvalState & state, Env & env, const AttrPath & a

void ExprSelect::eval(EvalState & state, Env & env, Value & v)
{
ZoneScopedN("select");
TRACY_TRACE(state, this)
Value vTmp;
PosIdx pos2;
Value * vAttrs = &vTmp;
Expand Down Expand Up @@ -1438,6 +1452,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)

void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v)
{
TRACY_TRACE(state, this)
Value vTmp;
Value * vAttrs = &vTmp;

Expand All @@ -1463,6 +1478,7 @@ void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v)

void ExprLambda::eval(EvalState & state, Env & env, Value & v)
{
TRACY_TRACE(state, this)
v.mkLambda(&env, this);
}

Expand Down Expand Up @@ -1720,6 +1736,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &

void ExprCall::eval(EvalState & state, Env & env, Value & v)
{
TRACY_TRACE(state, this);
Value vFun;
fun->eval(state, env, vFun);

Expand Down Expand Up @@ -1797,6 +1814,7 @@ values, or passed explicitly with '--arg' or '--argstr'. See

void ExprWith::eval(EvalState & state, Env & env, Value & v)
{
TRACY_TRACE(state, this);
Env & env2(state.allocEnv(1));
env2.up = &env;
env2.values[0] = attrs->maybeThunk(state, env);
Expand All @@ -1807,13 +1825,15 @@ void ExprWith::eval(EvalState & state, Env & env, Value & v)

void ExprIf::eval(EvalState & state, Env & env, Value & v)
{
TRACY_TRACE(state, this);
// We cheat in the parser, and pass the position of the condition as the position of the if itself.
(state.evalBool(env, cond, pos, "while evaluating a branch condition") ? then : else_)->eval(state, env, v);
}


void ExprAssert::eval(EvalState & state, Env & env, Value & v)
{
TRACY_TRACE(state, this);
if (!state.evalBool(env, cond, pos, "in the condition of the assert statement")) {
std::ostringstream out;
cond->show(state.symbols, out);
Expand All @@ -1825,12 +1845,14 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v)

void ExprOpNot::eval(EvalState & state, Env & env, Value & v)
{
TRACY_TRACE(state, this);
v.mkBool(!state.evalBool(env, e, getPos(), "in the argument of the not operator")); // XXX: FIXME: !
}


void ExprOpEq::eval(EvalState & state, Env & env, Value & v)
{
TRACY_TRACE(state, this);
Value v1; e1->eval(state, env, v1);
Value v2; e2->eval(state, env, v2);
v.mkBool(state.eqValues(v1, v2, pos, "while testing two values for equality"));
Expand All @@ -1839,6 +1861,7 @@ void ExprOpEq::eval(EvalState & state, Env & env, Value & v)

void ExprOpNEq::eval(EvalState & state, Env & env, Value & v)
{
TRACY_TRACE(state, this);
Value v1; e1->eval(state, env, v1);
Value v2; e2->eval(state, env, v2);
v.mkBool(!state.eqValues(v1, v2, pos, "while testing two values for inequality"));
Expand All @@ -1847,6 +1870,7 @@ void ExprOpNEq::eval(EvalState & state, Env & env, Value & v)

void ExprOpAnd::eval(EvalState & state, Env & env, Value & v)
{
TRACY_TRACE(state, this);
v.mkBool(state.evalBool(env, e1, pos, "in the left operand of the AND (&&) operator") && state.evalBool(env, e2, pos, "in the right operand of the AND (&&) operator"));
}

Expand All @@ -1859,12 +1883,14 @@ void ExprOpOr::eval(EvalState & state, Env & env, Value & v)

void ExprOpImpl::eval(EvalState & state, Env & env, Value & v)
{
TRACY_TRACE(state, this);
v.mkBool(!state.evalBool(env, e1, pos, "in the left operand of the IMPL (->) operator") || state.evalBool(env, e2, pos, "in the right operand of the IMPL (->) operator"));
}


void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v)
{
TRACY_TRACE(state, this);
Value v1, v2;
state.evalAttrs(env, e1, v1, pos, "in the left operand of the update (//) operator");
state.evalAttrs(env, e2, v2, pos, "in the right operand of the update (//) operator");
Expand Down Expand Up @@ -1903,6 +1929,7 @@ void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v)

void ExprOpConcatLists::eval(EvalState & state, Env & env, Value & v)
{
TRACY_TRACE(state, this);
Value v1; e1->eval(state, env, v1);
Value v2; e2->eval(state, env, v2);
Value * lists[2] = { &v1, &v2 };
Expand Down Expand Up @@ -1941,6 +1968,7 @@ void EvalState::concatLists(Value & v, size_t nrLists, Value * * lists, const Po

void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
{
TRACY_TRACE(state, this);
NixStringContext context;
std::vector<BackedStringView> s;
size_t sSize = 0;
Expand Down Expand Up @@ -2033,12 +2061,14 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)

void ExprPos::eval(EvalState & state, Env & env, Value & v)
{
TRACY_TRACE(state, this);
state.mkPos(v, pos);
}


void ExprBlackHole::eval(EvalState & state, Env & env, Value & v)
{
TRACY_TRACE(state, this);
state.error("infinite recursion encountered")
.debugThrow<InfiniteRecursionError>();
}
Expand Down
3 changes: 2 additions & 1 deletion src/libexpr/local.mk
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ libexpr_SOURCES := \
$(d)/parser-tab.cc \
$(d)/tracy/public/TracyClient.cpp

libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libmain -I src/libexpr -I src/libexpr/tracy/public
libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libmain -I src/libexpr -I src/libexpr/tracy/public -DTRACY_ENABLE=1

libexpr_LIBS = libutil libstore libfetchers

Expand Down Expand Up @@ -49,3 +49,4 @@ $(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh
$(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh $(d)/flake/call-flake.nix.gen.hh

$(buildprefix)src/libexpr/primops/fromTOML.o: ERROR_SWITCH_ENUM =
$(buildprefix)src/libexpr/tracy/public/TracyClient.o: ERROR_SWITCH_ENUM =
20 changes: 20 additions & 0 deletions src/libexpr/nixexpr.hh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,7 @@ struct Expr
virtual Value * maybeThunk(EvalState & state, Env & env);
virtual void setName(Symbol name);
virtual PosIdx getPos() const { return noPos; }
virtual const char* showExprType() const { return "undefined"; }
};

#define COMMON_METHODS \
Expand All @@ -168,6 +169,7 @@ struct ExprInt : Expr
Value v;
ExprInt(NixInt n) { v.mkInt(n); };
Value * maybeThunk(EvalState & state, Env & env) override;
const char* showExprType() const { return "int"; }
COMMON_METHODS
};

Expand All @@ -176,6 +178,7 @@ struct ExprFloat : Expr
Value v;
ExprFloat(NixFloat nf) { v.mkFloat(nf); };
Value * maybeThunk(EvalState & state, Env & env) override;
const char* showExprType() const { return "float"; }
COMMON_METHODS
};

Expand All @@ -185,6 +188,7 @@ struct ExprString : Expr
Value v;
ExprString(std::string &&s) : s(std::move(s)) { v.mkString(this->s.data()); };
Value * maybeThunk(EvalState & state, Env & env) override;
const char* showExprType() const { return "string"; }
COMMON_METHODS
};

Expand All @@ -198,6 +202,7 @@ struct ExprPath : Expr
v.mkPath(&*accessor, this->s.c_str());
}
Value * maybeThunk(EvalState & state, Env & env) override;
const char* showExprType() const { return "path"; }
COMMON_METHODS
};

Expand Down Expand Up @@ -229,6 +234,7 @@ struct ExprVar : Expr
ExprVar(const PosIdx & pos, Symbol name) : pos(pos), name(name) { };
Value * maybeThunk(EvalState & state, Env & env) override;
PosIdx getPos() const override { return pos; }
const char* showExprType() const { return "var"; }
COMMON_METHODS
};

Expand All @@ -240,6 +246,7 @@ struct ExprSelect : Expr
ExprSelect(const PosIdx & pos, Expr * e, AttrPath attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(std::move(attrPath)) { };
ExprSelect(const PosIdx & pos, Expr * e, Symbol name) : pos(pos), e(e), def(0) { attrPath.push_back(AttrName(name)); };
PosIdx getPos() const override { return pos; }
const char* showExprType() const { return "select"; }
COMMON_METHODS
};

Expand All @@ -249,6 +256,7 @@ struct ExprOpHasAttr : Expr
AttrPath attrPath;
ExprOpHasAttr(Expr * e, AttrPath attrPath) : e(e), attrPath(std::move(attrPath)) { };
PosIdx getPos() const override { return e->getPos(); }
const char* showExprType() const { return "op_has_attr"; }
COMMON_METHODS
};

Expand Down Expand Up @@ -278,13 +286,15 @@ struct ExprAttrs : Expr
ExprAttrs(const PosIdx &pos) : recursive(false), pos(pos) { };
ExprAttrs() : recursive(false) { };
PosIdx getPos() const override { return pos; }
const char* showExprType() const { return "attrs"; }
COMMON_METHODS
};

struct ExprList : Expr
{
std::vector<Expr *> elems;
ExprList() { };
const char* showExprType() const { return "list"; }
COMMON_METHODS
Value * maybeThunk(EvalState & state, Env & env) override;

Expand Down Expand Up @@ -345,6 +355,7 @@ struct ExprLambda : Expr
std::string showNamePos(const EvalState & state) const;
inline bool hasFormals() const { return formals != nullptr; }
PosIdx getPos() const override { return pos; }
const char* showExprType() const { return "lambda"; }
COMMON_METHODS
};

Expand All @@ -357,6 +368,7 @@ struct ExprCall : Expr
: fun(fun), args(args), pos(pos)
{ }
PosIdx getPos() const override { return pos; }
const char* showExprType() const { return "call"; }
COMMON_METHODS
};

Expand All @@ -365,6 +377,7 @@ struct ExprLet : Expr
ExprAttrs * attrs;
Expr * body;
ExprLet(ExprAttrs * attrs, Expr * body) : attrs(attrs), body(body) { };
const char* showExprType() const { return "let"; }
COMMON_METHODS
};

Expand All @@ -376,6 +389,7 @@ struct ExprWith : Expr
ExprWith * parentWith;
ExprWith(const PosIdx & pos, Expr * attrs, Expr * body) : pos(pos), attrs(attrs), body(body) { };
PosIdx getPos() const override { return pos; }
const char* showExprType() const { return "with"; }
COMMON_METHODS
};

Expand All @@ -385,6 +399,7 @@ struct ExprIf : Expr
Expr * cond, * then, * else_;
ExprIf(const PosIdx & pos, Expr * cond, Expr * then, Expr * else_) : pos(pos), cond(cond), then(then), else_(else_) { };
PosIdx getPos() const override { return pos; }
const char* showExprType() const { return "if"; }
COMMON_METHODS
};

Expand All @@ -394,6 +409,7 @@ struct ExprAssert : Expr
Expr * cond, * body;
ExprAssert(const PosIdx & pos, Expr * cond, Expr * body) : pos(pos), cond(cond), body(body) { };
PosIdx getPos() const override { return pos; }
const char* showExprType() const { return "assert"; }
COMMON_METHODS
};

Expand Down Expand Up @@ -422,6 +438,7 @@ struct ExprOpNot : Expr
} \
void eval(EvalState & state, Env & env, Value & v) override; \
PosIdx getPos() const override { return pos; } \
const char* showExprType() const { return #name; } \
};

MakeBinOp(ExprOpEq, "==")
Expand All @@ -440,6 +457,7 @@ struct ExprConcatStrings : Expr
ExprConcatStrings(const PosIdx & pos, bool forceString, std::vector<std::pair<PosIdx, Expr *>> * es)
: pos(pos), forceString(forceString), es(es) { };
PosIdx getPos() const override { return pos; }
const char* showExprType() const { return "concat_strings"; }
COMMON_METHODS
};

Expand All @@ -448,6 +466,7 @@ struct ExprPos : Expr
PosIdx pos;
ExprPos(const PosIdx & pos) : pos(pos) { };
PosIdx getPos() const override { return pos; }
const char* showExprType() const { return "pos"; }
COMMON_METHODS
};

Expand All @@ -457,6 +476,7 @@ struct ExprBlackHole : Expr
void show(const SymbolTable & symbols, std::ostream & str) const override {}
void eval(EvalState & state, Env & env, Value & v) override;
void bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env) override {}
const char* showExprType() const { return "blackhole"; }
};

extern ExprBlackHole eBlackHole;
Expand Down