Implement var statement pre-scanning. (#3103)

The patch also checks whether pre-scanning is successful when scanning is,
so no need for explicit pre-scanner checks anymore.

JerryScript-DCO-1.0-Signed-off-by: Zoltan Herczeg [email protected]

jerry-core/parser/js/js-parser-internal.h

@@ -86,6 +86,9 @@ typedef enum
  PARSER_MODULE_STORE_IDENT = (1u << 26), /**< store identifier of the current export statement */
  PARSER_IS_EVAL = (1u << 27), /**< eval code */
 #endif /* ENABLED (JERRY_ES2015_MODULE_SYSTEM) */
+#ifndef JERRY_NDEBUG
+ PARSER_SCANNING_SUCCESSFUL = (1u << 30), /**< scanning process was successful */
+#endif /* !JERRY_NDEBUG */
 } parser_general_flags_t;
 
 /**

jerry-core/parser/js/js-parser.c

@@ -2544,6 +2544,10 @@ parser_parse_source (const uint8_t *arg_list_p, /**< function argument list */
  compiled_code_p = parser_post_processing (&context);
  parser_list_free (&context.literal_pool);
 
+#ifndef JERRY_NDEBUG
+ JERRY_ASSERT (context.status_flags & PARSER_SCANNING_SUCCESSFUL);
+#endif /* !JERRY_NDEBUG */
+
 #if ENABLED (JERRY_PARSER_DUMP_BYTE_CODE)
  if (context.is_show_opcodes)
  {

jerry-core/parser/js/js-scanner-internal.h

@@ -0,0 +1,73 @@
+/* Copyright JS Foundation and other contributors, http:https://js.foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http:https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef JS_SCANNER_INTERNAL_H
+#define JS_SCANNER_INTERNAL_H
+
+/** \addtogroup parser Parser
+ * @{
+ *
+ * \addtogroup jsparser JavaScript
+ * @{
+ *
+ * \addtogroup jsparser_scanner Scanner
+ * @{
+ */
+
+/**
+ * Generic descriptor which stores only the start position.
+ */
+typedef struct
+{
+ const uint8_t *source_p; /**< start source byte */
+} scanner_source_start_t;
+
+/**
+ * For statement descriptor.
+ */
+typedef struct
+{
+ /** shared fields of for statements */
+ union
+ {
+ const uint8_t *source_p; /**< start source byte */
+ scanner_for_info_t *for_info_p; /**< for info */
+ } u;
+} scanner_for_statement_t;
+
+/**
+ * Switch statement descriptor.
+ */
+typedef struct
+{
+ scanner_case_info_t **last_case_p; /**< last case info */
+} scanner_switch_statement_t;
+
+/**
+ * Scanner context.
+ */
+typedef struct
+{
+ uint8_t mode; /**< scanner mode */
+ scanner_switch_statement_t active_switch_statement; /**< currently active switch statement */
+} scanner_context_t;
+
+/**
+ * @}
+ * @}
+ * @}
+ */
+
+#endif /* !JS_SCANNER_INTERNAL_H */

jerry-core/parser/js/js-scanner-util.c

@@ -14,6 +14,7 @@
  */
 
 #include "js-parser-internal.h"
+#include "js-scanner-internal.h"
 
 #if ENABLED (JERRY_PARSER)
 

jerry-core/parser/js/js-scanner.c

@@ -14,6 +14,7 @@
  */
 
 #include "js-parser-internal.h"
+#include "js-scanner-internal.h"
 #include "lit-char-helpers.h"
 
 #if ENABLED (JERRY_PARSER)
@@ -41,6 +42,7 @@ typedef enum
  SCAN_MODE_POST_PRIMARY_EXPRESSION, /**< scanning post primary expression */
  SCAN_MODE_PRIMARY_EXPRESSION_END, /**< scanning primary expression end */
  SCAN_MODE_STATEMENT, /**< scanning statement */
+ SCAN_MODE_VAR_STATEMENT, /**< scanning var statement */
  SCAN_MODE_FUNCTION_ARGUMENTS, /**< scanning function arguments */
 #if ENABLED (JERRY_ES2015_FUNCTION_PARAMETER_INITIALIZER)
  SCAN_MODE_CONTINUE_FUNCTION_ARGUMENTS, /**< continue scanning function arguments */
@@ -66,6 +68,9 @@ typedef enum
  SCAN_STACK_PAREN_EXPRESSION, /**< parent expression group */
  SCAN_STACK_PAREN_STATEMENT, /**< parent statement group */
  SCAN_STACK_WHILE_START, /**< start of "while" iterator */
+ /* The SCANNER_IS_FOR_START macro needs to be updated when the following constants are reordered. */
+ SCAN_STACK_VAR, /**< var statement */
+ SCAN_STACK_FOR_VAR_START, /**< start of "for" iterator with var statement */
  SCAN_STACK_FOR_START, /**< start of "for" iterator */
  SCAN_STACK_FOR_CONDITION, /**< condition part of "for" iterator */
  SCAN_STACK_FOR_EXPRESSION, /**< expression part of "for" iterator */
@@ -94,41 +99,19 @@ typedef enum
 } scan_stack_modes_t;
 
 /**
- * Generic descriptor which stores only the start position.
+ * Checks whether the stack top is a for statement start.
  */
-typedef struct
-{
- const uint8_t *source_p; /**< start source byte */
-} scanner_source_start_t;
-
-/**
- * For statement descriptor.
- */
-typedef struct
-{
- union
- {
- const uint8_t *source_p; /**< start source byte */
- scanner_for_info_t *for_info_p; /**< for info */
- } u;
-} scanner_for_statement_t;
+#define SCANNER_IS_FOR_START(stack_top) \
+ ((stack_top) >= SCAN_STACK_FOR_VAR_START && (stack_top) <= SCAN_STACK_FOR_START)
 
 /**
- * Switch statement descriptor.
+ * Checks whether token type is "of".
  */
-typedef struct
-{
- scanner_case_info_t **last_case_p; /**< last case info */
-} scanner_switch_statement_t;
-
-/**
- * Scanner context.
- */
-typedef struct
-{
- uint8_t mode; /**< scanner mode */
- scanner_switch_statement_t active_switch_statement; /**< currently active switch statement */
-} scanner_context_t;
+#if ENABLED (JERRY_ES2015_FOR_OF)
+#define SCANNER_IDENTIFIER_IS_OF() (lexer_compare_literal_to_identifier (context_p, "of", 2))
+#else
+#define SCANNER_IDENTIFIER_IS_OF() (false)
+#endif /* ENABLED (JERRY_ES2015_FOR_OF) */
 
 #if ENABLED (JERRY_ES2015_ARROW_FUNCTION)
 
@@ -401,6 +384,12 @@ scanner_scan_primary_expression_end (parser_context_t *context_p, /**< context *
  scanner_context_p->mode = SCAN_MODE_PROPERTY_NAME;
  return true;
  }
+ case SCAN_STACK_VAR:
+ case SCAN_STACK_FOR_VAR_START:
+ {
+ scanner_context_p->mode = SCAN_MODE_VAR_STATEMENT;
+ return false;
+ }
 #if ENABLED (JERRY_ES2015_FUNCTION_PARAMETER_INITIALIZER)
  case SCAN_STACK_FUNCTION_PARAMETERS:
  {
@@ -454,14 +443,24 @@ scanner_scan_primary_expression_end (parser_context_t *context_p, /**< context *
  }
 
  if (LEXER_IS_BINARY_OP_TOKEN (type)
- && (type != LEXER_KEYW_IN || stack_top != SCAN_STACK_FOR_START))
+ && (type != LEXER_KEYW_IN || !SCANNER_IS_FOR_START (stack_top)))
  {
  scanner_context_p->mode = SCAN_MODE_PRIMARY_EXPRESSION;
  return false;
  }
 
  switch (stack_top)
  {
+ case SCAN_STACK_VAR:
+ {
+ parser_stack_pop_uint8 (context_p);
+
+ if (type == LEXER_EOS)
+ {
+ return true;
+ }
+ /* FALLTHRU */
+ }
  case SCAN_STACK_HEAD:
  case SCAN_STACK_BLOCK_STATEMENT:
  case SCAN_STACK_FUNCTION_STATEMENT:
@@ -527,13 +526,10 @@ scanner_scan_primary_expression_end (parser_context_t *context_p, /**< context *
  scanner_context_p->mode = SCAN_MODE_STATEMENT;
  return false;
  }
+ case SCAN_STACK_FOR_VAR_START:
  case SCAN_STACK_FOR_START:
  {
- if (type == LEXER_KEYW_IN
-#if ENABLED (JERRY_ES2015_FOR_OF)
- || lexer_compare_literal_to_identifier (context_p, "of", 2)
-#endif /* ENABLED (JERRY_ES2015_FOR_OF) */
- || false)
+ if (type == LEXER_KEYW_IN || SCANNER_IDENTIFIER_IS_OF ())
  {
  scanner_for_statement_t for_statement;
 
@@ -869,26 +865,36 @@ scanner_scan_statement (parser_context_t *context_p, /**< context */
 
  scanner_for_statement_t for_statement;
  for_statement.u.source_p = context_p->source_p;
-
- parser_stack_push (context_p, &for_statement, sizeof (scanner_for_statement_t));
- parser_stack_push_uint8 (context_p, SCAN_STACK_FOR_START);
+ uint8_t stack_mode = SCAN_STACK_FOR_START;
 
  lexer_next_token (context_p);
+ scanner_context_p->mode = SCAN_MODE_PRIMARY_EXPRESSION;
 
- if (context_p->token.type == LEXER_SEMICOLON)
+ switch (context_p->token.type)
  {
- scanner_context_p->mode = SCAN_MODE_PRIMARY_EXPRESSION_END;
- return true;
+ case LEXER_SEMICOLON:
+ {
+ scanner_context_p->mode = SCAN_MODE_PRIMARY_EXPRESSION_END;
+ break;
+ }
+ case LEXER_KEYW_VAR:
+ {
+ scanner_context_p->mode = SCAN_MODE_VAR_STATEMENT;
+ stack_mode = SCAN_STACK_FOR_VAR_START;
+ break;
+ }
  }
 
- scanner_context_p->mode = SCAN_MODE_PRIMARY_EXPRESSION;
- if (context_p->token.type == LEXER_KEYW_VAR)
- {
- return false;
- }
- return true;
+ parser_stack_push (context_p, &for_statement, sizeof (scanner_for_statement_t));
+ parser_stack_push_uint8 (context_p, stack_mode);
+ return (stack_mode == SCAN_STACK_FOR_START);
  }
  case LEXER_KEYW_VAR:
+ {
+ parser_stack_push_uint8 (context_p, SCAN_STACK_VAR);
+ scanner_context_p->mode = SCAN_MODE_VAR_STATEMENT;
+ return false;
+ }
  case LEXER_KEYW_THROW:
  {
  scanner_context_p->mode = SCAN_MODE_PRIMARY_EXPRESSION;
@@ -1351,6 +1357,10 @@ scanner_scan_all (parser_context_t *context_p) /**< context */
 
  if (type == LEXER_EOS)
  {
+#ifndef JERRY_NDEBUG
+ context_p->status_flags |= PARSER_SCANNING_SUCCESSFUL;
+#endif /* !JERRY_NDEBUG */
+
  if (stack_top == SCAN_STACK_HEAD)
  {
  break;
@@ -1498,6 +1508,59 @@ scanner_scan_all (parser_context_t *context_p) /**< context */
  }
  break;
  }
+ case SCAN_MODE_VAR_STATEMENT:
+ {
+ if (type != LEXER_LITERAL
+ || context_p->token.lit_location.type != LEXER_IDENT_LITERAL)
+ {
+ scanner_raise_error (context_p);
+ }
+
+ lexer_next_token (context_p);
+
+ switch (context_p->token.type)
+ {
+ case LEXER_ASSIGN:
+ {
+ scanner_context.mode = SCAN_MODE_PRIMARY_EXPRESSION;
+ /* FALLTHRU */
+ }
+ case LEXER_COMMA:
+ {
+ lexer_next_token (context_p);
+ continue;
+ }
+ case LEXER_SEMICOLON:
+ case LEXER_RIGHT_BRACE:
+ {
+ scanner_context.mode = SCAN_MODE_PRIMARY_EXPRESSION_END;
+ continue;
+ }
+ }
+
+ if (SCANNER_IS_FOR_START (stack_top))
+ {
+ if (context_p->token.type != LEXER_KEYW_IN && !SCANNER_IDENTIFIER_IS_OF ())
+ {
+ scanner_raise_error (context_p);
+ }
+
+ scanner_context.mode = SCAN_MODE_PRIMARY_EXPRESSION_END;
+ continue;
+ }
+
+ if (context_p->token.type != LEXER_EOS
+ && !(context_p->token.flags & LEXER_WAS_NEWLINE))
+ {
+ scanner_raise_error (context_p);
+ }
+
+ JERRY_ASSERT (stack_top == SCAN_STACK_VAR);
+
+ scanner_context.mode = SCAN_MODE_STATEMENT;
+ parser_stack_pop_uint8 (context_p);
+ continue;
+ }
  case SCAN_MODE_FUNCTION_ARGUMENTS:
  {
  JERRY_ASSERT (stack_top == SCAN_STACK_FUNCTION_STATEMENT

tests/jerry/es2015/class.js

@@ -236,6 +236,3 @@ assert (G.get() == 11);
 assert (G.set() == 12);
 G.constructor = 30;
 assert (G.constructor === 30);
-
-// Pre-scanner regression test
-for (var tmp in {}) ;

tests/jerry/es2015/function-rest-parameter.js

@@ -78,6 +78,3 @@ assert (g2 () === 11);
 assert (g2 (1) === 3);
 assert (g2 (1, 2) === 3);
 assert (g2 (1, 2, 3) === 4);
-
-// Pre-scanner regression test
-for (var tmp in {}) ;

tests/jerry/es2015/module-export-01.js

@@ -39,6 +39,3 @@ assert (x === 42);
 assert (f(1) === 1);
 var dog = new Dog("Pluto")
 assert(dog.speak() === "Pluto barks.")
-
-// Pre-scanner regression test
-for (var tmp in {}) ;

tests/jerry/es2015/module-export-02.js

@@ -18,6 +18,3 @@ export {aa,} from "module-export-01.js";
 export {bb as b_, cc as c_} from "module-export-01.js";
 export * from "module-export-01.js";
 export default function () {return "default"};
-
-// Pre-scanner regression test
-for (var tmp in {}) ;

tests/jerry/es2015/module-export-03.js

@@ -24,6 +24,3 @@ export default class {
 }
 
 export * from "module-export-02.js"
-
-// Pre-scanner regression test
-for (var tmp in {}) ;

tests/jerry/es2015/module-export-04.js

@@ -15,6 +15,3 @@
 
 export var x = 41
 export default a = "str"
-
-// Pre-scanner regression test
-for (var tmp in {}) ;