Split out infix and prefix operators to separate node kinds. (carbon-…

…language#3481)

This leaves a single state for each in the expr loop. I was trying to
think through ways to have per-token states, but they felt sort of
bulky.

Note this is more verbose: but I think the long-term is going to be that
when we start wanting to add handlers, we're going to need to switch to
different names based on the token found. As a consequence, the parse
state logic will end up diverging a little, and we'll just want to align
towards boilerplate handlers.

Short-term, this opens up a path for saying that each parse node
corresponds to precisely one token in success states, and separates out
what were becoming big handler functions in check.

toolchain/check/node_stack.h

@@ -330,12 +330,10 @@ class NodeStack {
  case Parse::NodeKind::IfExprThen:
  case Parse::NodeKind::IfExprElse:
  case Parse::NodeKind::IndexExpr:
- case Parse::NodeKind::InfixOperator:
  case Parse::NodeKind::MemberAccessExpr:
  case Parse::NodeKind::PackageExpr:
  case Parse::NodeKind::ParenExpr:
- case Parse::NodeKind::PostfixOperator:
- case Parse::NodeKind::PrefixOperator:
+ case Parse::NodeKind::PostfixOperatorStar:
  case Parse::NodeKind::ReturnType:
  case Parse::NodeKind::SelfTypeNameExpr:
  case Parse::NodeKind::SelfValueNameExpr:
@@ -386,6 +384,12 @@ class NodeStack {
  return IdKind::SoloParseNode;
 // Use x-macros to handle token cases.
 #define CARBON_PARSE_NODE_KIND(...)
+#define CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(Name, ...) \
+ case Parse::NodeKind::InfixOperator##Name: \
+ return IdKind::InstId;
+#define CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR(Name, ...) \
+ case Parse::NodeKind::PrefixOperator##Name: \
+ return IdKind::InstId;
 #define CARBON_PARSE_NODE_KIND_TOKEN_LITERAL(Name, ...) \
  case Parse::NodeKind::Name: \
  return IdKind::InstId;

toolchain/check/testdata/index/fail_negative_indexing.carbon

@@ -5,7 +5,7 @@
 // AUTOUPDATE
 
 var a: (i32, i32) = (12, 6);
-// CHECK:STDERR: fail_negative_indexing.carbon:[[@LINE+3]]:16: ERROR: Semantics TODO: `Handle -`.
+// CHECK:STDERR: fail_negative_indexing.carbon:[[@LINE+3]]:16: ERROR: Semantics TODO: `HandlePrefixOperatorMinus`.
 // CHECK:STDERR: var b: i32 = a[-10];
 // CHECK:STDERR: ^~~
 var b: i32 = a[-10];

toolchain/check/testdata/operators/fail_type_mismatch_once.carbon

@@ -7,7 +7,7 @@
 fn Main() -> i32 {
  // The following line has two mismatches, but after the first, it shouldn't
  // keep erroring.
- // CHECK:STDERR: fail_type_mismatch_once.carbon:[[@LINE+3]]:10: ERROR: Semantics TODO: `Handle +`.
+ // CHECK:STDERR: fail_type_mismatch_once.carbon:[[@LINE+3]]:10: ERROR: Semantics TODO: `HandleInfixOperatorPlus`.
  // CHECK:STDERR: return 12 + 3.4 + 12;
  // CHECK:STDERR: ^~~~~~~~
  return 12 + 3.4 + 12;

toolchain/check/testdata/operators/fail_unimplemented_op.carbon

@@ -5,7 +5,7 @@
 // AUTOUPDATE
 
 fn Main() -> i32 {
- // CHECK:STDERR: fail_unimplemented_op.carbon:[[@LINE+3]]:10: ERROR: Semantics TODO: `Handle +`.
+ // CHECK:STDERR: fail_unimplemented_op.carbon:[[@LINE+3]]:10: ERROR: Semantics TODO: `HandleInfixOperatorPlus`.
  // CHECK:STDERR: return 12 + 34;
  // CHECK:STDERR: ^~~~~~~
  return 12 + 34;

toolchain/parse/handle_expr.cpp

@@ -2,6 +2,7 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+#include "toolchain/lex/token_kind.h"
 #include "toolchain/parse/context.h"
 
 namespace Carbon::Parse {
@@ -48,7 +49,7 @@ auto HandleExpr(Context& context) -> void {
  context.PushState(State::IfExprFinish);
  context.PushState(State::IfExprFinishCondition);
  } else {
- context.PushStateForExprLoop(State::ExprLoopForPrefix,
+ context.PushStateForExprLoop(State::ExprLoopForPrefixOperator,
  state.ambient_precedence,
  *operator_precedence);
  }
@@ -279,44 +280,72 @@ auto HandleExprLoop(Context& context) -> void {
  break;
 
  default:
- state.state = State::ExprLoopForBinary;
+ state.state = State::ExprLoopForInfixOperator;
  break;
  }
 
  context.PushState(state);
  context.PushStateForExpr(operator_precedence);
  } else {
- context.AddNode(NodeKind::PostfixOperator, state.token, state.subtree_start,
- state.has_error);
+ context.AddNode(NodeKind::PostfixOperatorStar, state.token,
+ state.subtree_start, state.has_error);
  state.has_error = false;
  context.PushState(state);
  }
 }
 
 // Adds the operator node and returns the the main expression loop.
-static auto HandleExprLoopForOperator(Context& context, NodeKind node_kind)
- -> void {
- auto state = context.PopState();
-
+static auto HandleExprLoopForOperator(Context& context,
+ Context::StateStackEntry state,
+ NodeKind node_kind) -> void {
  context.AddNode(node_kind, state.token, state.subtree_start, state.has_error);
  state.has_error = false;
  context.PushState(state, State::ExprLoop);
 }
 
-auto HandleExprLoopForBinary(Context& context) -> void {
- HandleExprLoopForOperator(context, NodeKind::InfixOperator);
+auto HandleExprLoopForInfixOperator(Context& context) -> void {
+ auto state = context.PopState();
+
+ switch (auto token_kind = context.tokens().GetKind(state.token)) {
+#define CARBON_PARSE_NODE_KIND(...)
+#define CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(Name, ...) \
+ case Lex::TokenKind::Name: \
+ HandleExprLoopForOperator(context, state, NodeKind::InfixOperator##Name); \
+ break;
+#include "toolchain/parse/node_kind.def"
+
+ default:
+ CARBON_FATAL() << "Unexpected token kind for infix operator: "
+ << token_kind;
+ }
 }
 
-auto HandleExprLoopForPrefix(Context& context) -> void {
- HandleExprLoopForOperator(context, NodeKind::PrefixOperator);
+auto HandleExprLoopForPrefixOperator(Context& context) -> void {
+ auto state = context.PopState();
+
+ switch (auto token_kind = context.tokens().GetKind(state.token)) {
+#define CARBON_PARSE_NODE_KIND(...)
+#define CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR(Name, ...) \
+ case Lex::TokenKind::Name: \
+ HandleExprLoopForOperator(context, state, NodeKind::PrefixOperator##Name); \
+ break;
+#include "toolchain/parse/node_kind.def"
+
+ default:
+ CARBON_FATAL() << "Unexpected token kind for prefix operator: "
+ << token_kind;
+ }
 }
 
 auto HandleExprLoopForShortCircuitOperatorAsAnd(Context& context) -> void {
- HandleExprLoopForOperator(context, NodeKind::ShortCircuitOperatorAnd);
+ auto state = context.PopState();
+ HandleExprLoopForOperator(context, state, NodeKind::ShortCircuitOperatorAnd);
 }
 
 auto HandleExprLoopForShortCircuitOperatorAsOr(Context& context) -> void {
- HandleExprLoopForOperator(context, NodeKind::ShortCircuitOperatorOr);
+ auto state = context.PopState();
+
+ HandleExprLoopForOperator(context, state, NodeKind::ShortCircuitOperatorOr);
 }
 
 auto HandleIfExprFinishCondition(Context& context) -> void {

toolchain/parse/node_kind.def

@@ -17,6 +17,10 @@
 // - CARBON_PARSE_NODE_KIND_CHILD_COUNT(Name, ChildCount, LexTokenKinds)
 // Defines a parse node with a set number of children, often 0. This count
 // must be correct even when the node contains errors.
+// - CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(Name)
+// Defines a parse node for an infix operator, with the Name as token.
+// - CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR(Name)
+// Defines a parse node for a prefix operator, with the Name as token.
 // - CARBON_PARSE_NODE_KIND_TOKEN_LITERAL(Name, LexTokenKinds)
 // Defines a parse node that corresponds to a token that is a single-token
 // literal. The token is wrapped for LexTokenKinds.
@@ -53,6 +57,28 @@
  CARBON_PARSE_NODE_KIND(Name)
 #endif
 
+// This is expected to be used with something like:
+//
+// // Use x-macros to handle modifier cases.
+// #define CARBON_PARSE_NODE_KIND(...)
+// #define CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(Name, ...) <code>
+#ifndef CARBON_PARSE_NODE_KIND_INFIX_OPERATOR
+#define CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(Name) \
+ CARBON_PARSE_NODE_KIND_CHILD_COUNT(InfixOperator##Name, 2, \
+ CARBON_TOKEN(Name))
+#endif
+
+// This is expected to be used with something like:
+//
+// // Use x-macros to handle modifier cases.
+// #define CARBON_PARSE_NODE_KIND(...)
+// #define CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR(Name, ...) <code>
+#ifndef CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR
+#define CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR(Name) \
+ CARBON_PARSE_NODE_KIND_CHILD_COUNT(PrefixOperator##Name, 1, \
+ CARBON_TOKEN(Name))
+#endif
+
 // This is expected to be used with something like:
 //
 // // Use x-macros to handle literal cases.
@@ -511,70 +537,69 @@ CARBON_PARSE_NODE_KIND_TOKEN_LITERAL(TypeTypeLiteral, CARBON_TOKEN(Type))
 
 // clang-format off
 
-// A prefix operator:
+// A prefix operator, such as `not`:
 // _external_: expression
-// PrefixOperator
-CARBON_PARSE_NODE_KIND_CHILD_COUNT(PrefixOperator, 1,
- CARBON_TOKEN(Star)
- CARBON_TOKEN(Amp)
- CARBON_TOKEN(Not)
- CARBON_TOKEN(Minus)
- CARBON_TOKEN(MinusMinus)
- CARBON_TOKEN(PlusPlus)
- CARBON_TOKEN(Caret)
- CARBON_TOKEN(Const))
-
-// An infix operator:
+// PrefixOperator<name>
+CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR(Amp)
+CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR(Caret)
+CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR(Const)
+CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR(Not)
+CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR(Minus)
+CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR(MinusMinus)
+CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR(PlusPlus)
+CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR(Star)
+
+// An infix operator, such as `+`:
 // _external_: lhs expression
 // _external_: rhs expression
-// InfixOperator
-CARBON_PARSE_NODE_KIND_CHILD_COUNT(InfixOperator, 2,
- CARBON_TOKEN(Amp)
- CARBON_TOKEN(AmpEqual)
- CARBON_TOKEN(As)
- CARBON_TOKEN(Caret)
- CARBON_TOKEN(CaretEqual)
- CARBON_TOKEN(Equal)
- CARBON_TOKEN(EqualEqual)
- CARBON_TOKEN(ExclaimEqual)
- CARBON_TOKEN(Greater)
- CARBON_TOKEN(GreaterEqual)
- CARBON_TOKEN(GreaterGreater)
- CARBON_TOKEN(GreaterGreaterEqual)
- CARBON_TOKEN(Less)
- CARBON_TOKEN(LessEqual)
- CARBON_TOKEN(LessEqualGreater)
- CARBON_TOKEN(LessLess)
- CARBON_TOKEN(LessLessEqual)
- CARBON_TOKEN(Minus)
- CARBON_TOKEN(MinusEqual)
- CARBON_TOKEN(Percent)
- CARBON_TOKEN(PercentEqual)
- CARBON_TOKEN(Pipe)
- CARBON_TOKEN(PipeEqual)
- CARBON_TOKEN(Plus)
- CARBON_TOKEN(PlusEqual)
- CARBON_TOKEN(Slash)
- CARBON_TOKEN(SlashEqual)
- CARBON_TOKEN(Star)
- CARBON_TOKEN(StarEqual))
+// InfixOperator<name>
+CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(Amp)
+CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(AmpEqual)
+CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(As)
+CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(Caret)
+CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(CaretEqual)
+CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(Equal)
+CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(EqualEqual)
+CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(ExclaimEqual)
+CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(Greater)
+CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(GreaterEqual)
+CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(GreaterGreater)
+CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(GreaterGreaterEqual)
+CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(Less)
+CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(LessEqual)
+CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(LessEqualGreater)
+CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(LessLess)
+CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(LessLessEqual)
+CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(Minus)
+CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(MinusEqual)
+CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(Percent)
+CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(PercentEqual)
+CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(Pipe)
+CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(PipeEqual)
+CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(Plus)
+CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(PlusEqual)
+CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(Slash)
+CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(SlashEqual)
+CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(Star)
+CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(StarEqual)
 
 // clang-format on
 
-// A short-circuiting infix operator:
+// A short-circuiting infix operator, such as `and`:
 // _external_: expression
 // ShortCircuitOperand(And|Or)
 // _external_: expression
 // ShortCircuitOperand(And|Or)
 CARBON_PARSE_NODE_KIND_CHILD_COUNT(ShortCircuitOperandAnd, 1, CARBON_TOKEN(And))
 CARBON_PARSE_NODE_KIND_CHILD_COUNT(ShortCircuitOperandOr, 1, CARBON_TOKEN(Or))
-CARBON_PARSE_NODE_KIND_CHILD_COUNT(ShortCircuitOperatorAnd, 2, CARBON_TOKEN(And))
+CARBON_PARSE_NODE_KIND_CHILD_COUNT(ShortCircuitOperatorAnd, 2,
+ CARBON_TOKEN(And))
 CARBON_PARSE_NODE_KIND_CHILD_COUNT(ShortCircuitOperatorOr, 2, CARBON_TOKEN(Or))
 
-// A postfix operator:
+// A postfix operator, currently only `*`:
 // _external_: expression
-// PostfixOperator
-CARBON_PARSE_NODE_KIND_CHILD_COUNT(PostfixOperator, 1, CARBON_TOKEN(Star))
+// PostfixOperatorStar
+CARBON_PARSE_NODE_KIND_CHILD_COUNT(PostfixOperatorStar, 1, CARBON_TOKEN(Star))
 
 // `if` expression + `then` + `else`:
 // _external_: expression
@@ -747,6 +772,8 @@ CARBON_PARSE_NODE_KIND_BRACKET(NamedConstraintDecl, NamedConstraintIntroducer,
 #undef CARBON_PARSE_NODE_KIND
 #undef CARBON_PARSE_NODE_KIND_BRACKET
 #undef CARBON_PARSE_NODE_KIND_CHILD_COUNT
+#undef CARBON_PARSE_NODE_KIND_INFIX_OPERATOR
+#undef CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR
 #undef CARBON_PARSE_NODE_KIND_TOKEN_LITERAL
 #undef CARBON_PARSE_NODE_KIND_TOKEN_MODIFIER
 #undef CARBON_TOKEN

toolchain/parse/state.def

@@ -501,15 +501,15 @@ CARBON_PARSE_STATE(ExprLoop)
 // expr <infix operator> expr ...
 // ^
 // 1. ExprLoop
-CARBON_PARSE_STATE(ExprLoopForBinary)
+CARBON_PARSE_STATE(ExprLoopForInfixOperator)
 
 // Completes an ExprLoop pass by adding a prefix operator, then goes back
 // to ExprLoop.
 //
 // <prefix operator> expr ...
 // ^
 // 1. ExprLoop
-CARBON_PARSE_STATE(ExprLoopForPrefix)
+CARBON_PARSE_STATE(ExprLoopForPrefixOperator)
 
 // Completes an ExprLoop pass by adding a short circuit operator, then goes back
 // to ExprLoop.

toolchain/parse/testdata/class/fn_definitions.carbon

@@ -57,7 +57,7 @@ class Foo {
 // CHECK:STDOUT: {kind: 'ImplicitParamListStart', text: '['},
 // CHECK:STDOUT: {kind: 'SelfValueName', text: 'self'},
 // CHECK:STDOUT: {kind: 'SelfTypeNameExpr', text: 'Self'},
-// CHECK:STDOUT: {kind: 'PostfixOperator', text: '*', subtree_size: 2},
+// CHECK:STDOUT: {kind: 'PostfixOperatorStar', text: '*', subtree_size: 2},
 // CHECK:STDOUT: {kind: 'BindingPattern', text: ':', subtree_size: 4},
 // CHECK:STDOUT: {kind: 'Address', text: 'addr', subtree_size: 5},
 // CHECK:STDOUT: {kind: 'ImplicitParamList', text: ']', subtree_size: 7},
@@ -68,7 +68,7 @@ class Foo {
 // CHECK:STDOUT: {kind: 'IdentifierName', text: 'x'},
 // CHECK:STDOUT: {kind: 'PointerMemberAccessExpr', text: '->', subtree_size: 3},
 // CHECK:STDOUT: {kind: 'IntLiteral', text: '1'},
-// CHECK:STDOUT: {kind: 'InfixOperator', text: '=', subtree_size: 5},
+// CHECK:STDOUT: {kind: 'InfixOperatorEqual', text: '=', subtree_size: 5},
 // CHECK:STDOUT: {kind: 'ExprStatement', text: ';', subtree_size: 6},
 // CHECK:STDOUT: {kind: 'FunctionDefinition', text: '}', subtree_size: 19},
 // CHECK:STDOUT: {kind: 'ClassDefinition', text: '}', subtree_size: 57},

toolchain/parse/testdata/function/declaration/addr.carbon

@@ -14,7 +14,7 @@ fn foo(addr a: i32*);
 // CHECK:STDOUT: {kind: 'TuplePatternStart', text: '('},
 // CHECK:STDOUT: {kind: 'IdentifierName', text: 'a'},
 // CHECK:STDOUT: {kind: 'IntTypeLiteral', text: 'i32'},
-// CHECK:STDOUT: {kind: 'PostfixOperator', text: '*', subtree_size: 2},
+// CHECK:STDOUT: {kind: 'PostfixOperatorStar', text: '*', subtree_size: 2},
 // CHECK:STDOUT: {kind: 'BindingPattern', text: ':', subtree_size: 4},
 // CHECK:STDOUT: {kind: 'Address', text: 'addr', subtree_size: 5},
 // CHECK:STDOUT: {kind: 'TuplePattern', text: ')', subtree_size: 7},

toolchain/parse/testdata/function/definition/with_params.carbon

@@ -29,7 +29,7 @@ fn foo(bar: i64, baz: i64) {
 // CHECK:STDOUT: {kind: 'CallExprComma', text: ','},
 // CHECK:STDOUT: {kind: 'IdentifierNameExpr', text: 'bar'},
 // CHECK:STDOUT: {kind: 'IdentifierNameExpr', text: 'baz'},
-// CHECK:STDOUT: {kind: 'InfixOperator', text: '+', subtree_size: 3},
+// CHECK:STDOUT: {kind: 'InfixOperatorPlus', text: '+', subtree_size: 3},
 // CHECK:STDOUT: {kind: 'CallExpr', text: ')', subtree_size: 8},
 // CHECK:STDOUT: {kind: 'ExprStatement', text: ';', subtree_size: 9},
 // CHECK:STDOUT: {kind: 'FunctionDefinition', text: '}', subtree_size: 22},

toolchain/parse/testdata/generics/interface/default_fn.carbon

@@ -53,7 +53,7 @@ interface Foo {
 // CHECK:STDOUT: {kind: 'MemberAccessExpr', text: '.', subtree_size: 3},
 // CHECK:STDOUT: {kind: 'CallExprStart', text: '(', subtree_size: 4},
 // CHECK:STDOUT: {kind: 'IntLiteral', text: '1'},
-// CHECK:STDOUT: {kind: 'PrefixOperator', text: '-', subtree_size: 2},
+// CHECK:STDOUT: {kind: 'PrefixOperatorMinus', text: '-', subtree_size: 2},
 // CHECK:STDOUT: {kind: 'CallExpr', text: ')', subtree_size: 7},
 // CHECK:STDOUT: {kind: 'ReturnStatement', text: ';', subtree_size: 9},
 // CHECK:STDOUT: {kind: 'FunctionDefinition', text: '}', subtree_size: 23},