LibWeb+LibSyntax: Implement nested syntax highlighters

And use them to highlight javascript in HTML source.
This commit also changes how TextDocumentSpan::data is interpreted,
as it used to be an opaque pointer, but everyone stuffed an enum value
inside it, which made the values not unique to each highlighter;
that field is now a u64 serial id.
The syntax highlighters don't need to change their ways of stuffing
token types into that field, but a highlighter that calls another
nested highlighter needs to register the nested types for use with
token pairs.

Userland/Applications/Spreadsheet/CellSyntaxHighlighter.cpp

@@ -31,7 +31,7 @@ void CellSyntaxHighlighter::rehighlight(const Palette& palette)
  false,
  false,
  },
- nullptr,
+ (u64)-1,
  false);
 
  if (m_cell && m_cell->exception()) {
@@ -47,7 +47,7 @@ void CellSyntaxHighlighter::rehighlight(const Palette& palette)
  false,
  false,
  },
- nullptr,
+ (u64)-1,
  false });
  }
  m_client->do_update();

Userland/Libraries/LibCpp/SyntaxHighlighter.cpp

@@ -43,15 +43,15 @@ static Syntax::TextStyle style_for_token_type(Gfx::Palette const& palette, Cpp::
  }
 }
 
-bool SyntaxHighlighter::is_identifier(void* token) const
+bool SyntaxHighlighter::is_identifier(u64 token) const
 {
- auto cpp_token = static_cast<Cpp::Token::Type>(reinterpret_cast<size_t>(token));
+ auto cpp_token = static_cast<Cpp::Token::Type>(token);
  return cpp_token == Cpp::Token::Type::Identifier;
 }
 
-bool SyntaxHighlighter::is_navigatable(void* token) const
+bool SyntaxHighlighter::is_navigatable(u64 token) const
 {
- auto cpp_token = static_cast<Cpp::Token::Type>(reinterpret_cast<size_t>(token));
+ auto cpp_token = static_cast<Cpp::Token::Type>(token);
  return cpp_token == Cpp::Token::Type::IncludePath;
 }
 
@@ -72,7 +72,7 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
  span.attributes.color = style.color;
  span.attributes.bold = style.bold;
  span.is_skippable = token.type() == Cpp::Token::Type::Whitespace;
- span.data = reinterpret_cast<void*>(token.type());
+ span.data = static_cast<u64>(token.type());
  spans.append(span);
  }
  m_client->do_set_spans(move(spans));
@@ -83,20 +83,20 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
  m_client->do_update();
 }
 
-Vector<SyntaxHighlighter::MatchingTokenPair> SyntaxHighlighter::matching_token_pairs() const
+Vector<SyntaxHighlighter::MatchingTokenPair> SyntaxHighlighter::matching_token_pairs_impl() const
 {
  static Vector<SyntaxHighlighter::MatchingTokenPair> pairs;
  if (pairs.is_empty()) {
- pairs.append({ reinterpret_cast<void*>(Cpp::Token::Type::LeftCurly), reinterpret_cast<void*>(Cpp::Token::Type::RightCurly) });
- pairs.append({ reinterpret_cast<void*>(Cpp::Token::Type::LeftParen), reinterpret_cast<void*>(Cpp::Token::Type::RightParen) });
- pairs.append({ reinterpret_cast<void*>(Cpp::Token::Type::LeftBracket), reinterpret_cast<void*>(Cpp::Token::Type::RightBracket) });
+ pairs.append({ static_cast<u64>(Cpp::Token::Type::LeftCurly), static_cast<u64>(Cpp::Token::Type::RightCurly) });
+ pairs.append({ static_cast<u64>(Cpp::Token::Type::LeftParen), static_cast<u64>(Cpp::Token::Type::RightParen) });
+ pairs.append({ static_cast<u64>(Cpp::Token::Type::LeftBracket), static_cast<u64>(Cpp::Token::Type::RightBracket) });
  }
  return pairs;
 }
 
-bool SyntaxHighlighter::token_types_equal(void* token1, void* token2) const
+bool SyntaxHighlighter::token_types_equal(u64 token1, u64 token2) const
 {
- return static_cast<Cpp::Token::Type>(reinterpret_cast<size_t>(token1)) == static_cast<Cpp::Token::Type>(reinterpret_cast<size_t>(token2));
+ return static_cast<Cpp::Token::Type>(token1) == static_cast<Cpp::Token::Type>(token2);
 }
 
 SyntaxHighlighter::~SyntaxHighlighter()

Userland/Libraries/LibCpp/SyntaxHighlighter.h

@@ -15,15 +15,15 @@ class SyntaxHighlighter final : public Syntax::Highlighter {
  SyntaxHighlighter() { }
  virtual ~SyntaxHighlighter() override;
 
- virtual bool is_identifier(void*) const override;
- virtual bool is_navigatable(void*) const override;
+ virtual bool is_identifier(u64) const override;
+ virtual bool is_navigatable(u64) const override;
 
  virtual Syntax::Language language() const override { return Syntax::Language::Cpp; }
  virtual void rehighlight(Palette const&) override;
 
 protected:
- virtual Vector<MatchingTokenPair> matching_token_pairs() const override;
- virtual bool token_types_equal(void*, void*) const override;
+ virtual Vector<MatchingTokenPair> matching_token_pairs_impl() const override;
+ virtual bool token_types_equal(u64, u64) const override;
 };
 
 }

Userland/Libraries/LibGUI/GMLSyntaxHighlighter.cpp

@@ -33,9 +33,9 @@ static Syntax::TextStyle style_for_token_type(const Gfx::Palette& palette, GMLTo
  }
 }
 
-bool GMLSyntaxHighlighter::is_identifier(void* token) const
+bool GMLSyntaxHighlighter::is_identifier(u64 token) const
 {
- auto ini_token = static_cast<GUI::GMLToken::Type>(reinterpret_cast<size_t>(token));
+ auto ini_token = static_cast<GUI::GMLToken::Type>(token);
  return ini_token == GUI::GMLToken::Type::Identifier;
 }
 
@@ -54,7 +54,7 @@ void GMLSyntaxHighlighter::rehighlight(const Palette& palette)
  span.attributes.color = style.color;
  span.attributes.bold = style.bold;
  span.is_skippable = false;
- span.data = reinterpret_cast<void*>(token.m_type);
+ span.data = static_cast<u64>(token.m_type);
  spans.append(span);
  }
  m_client->do_set_spans(move(spans));
@@ -65,18 +65,18 @@ void GMLSyntaxHighlighter::rehighlight(const Palette& palette)
  m_client->do_update();
 }
 
-Vector<GMLSyntaxHighlighter::MatchingTokenPair> GMLSyntaxHighlighter::matching_token_pairs() const
+Vector<GMLSyntaxHighlighter::MatchingTokenPair> GMLSyntaxHighlighter::matching_token_pairs_impl() const
 {
  static Vector<MatchingTokenPair> pairs;
  if (pairs.is_empty()) {
- pairs.append({ reinterpret_cast<void*>(GMLToken::Type::LeftCurly), reinterpret_cast<void*>(GMLToken::Type::RightCurly) });
+ pairs.append({ static_cast<u64>(GMLToken::Type::LeftCurly), static_cast<u64>(GMLToken::Type::RightCurly) });
  }
  return pairs;
 }
 
-bool GMLSyntaxHighlighter::token_types_equal(void* token1, void* token2) const
+bool GMLSyntaxHighlighter::token_types_equal(u64 token1, u64 token2) const
 {
- return static_cast<GUI::GMLToken::Type>(reinterpret_cast<size_t>(token1)) == static_cast<GUI::GMLToken::Type>(reinterpret_cast<size_t>(token2));
+ return static_cast<GUI::GMLToken::Type>(token1) == static_cast<GUI::GMLToken::Type>(token2);
 }
 
 GMLSyntaxHighlighter::~GMLSyntaxHighlighter()

Userland/Libraries/LibGUI/GMLSyntaxHighlighter.h

@@ -15,14 +15,14 @@ class GMLSyntaxHighlighter final : public Syntax::Highlighter {
  GMLSyntaxHighlighter() { }
  virtual ~GMLSyntaxHighlighter() override;
 
- virtual bool is_identifier(void*) const override;
+ virtual bool is_identifier(u64) const override;
 
  virtual Syntax::Language language() const override { return Syntax::Language::GML; }
  virtual void rehighlight(const Palette&) override;
 
 protected:
- virtual Vector<MatchingTokenPair> matching_token_pairs() const override;
- virtual bool token_types_equal(void*, void*) const override;
+ virtual Vector<MatchingTokenPair> matching_token_pairs_impl() const override;
+ virtual bool token_types_equal(u64, u64) const override;
 };
 
 }

Userland/Libraries/LibGUI/INISyntaxHighlighter.cpp

@@ -32,9 +32,9 @@ static Syntax::TextStyle style_for_token_type(const Gfx::Palette& palette, IniTo
  }
 }
 
-bool IniSyntaxHighlighter::is_identifier(void* token) const
+bool IniSyntaxHighlighter::is_identifier(u64 token) const
 {
- auto ini_token = static_cast<GUI::IniToken::Type>(reinterpret_cast<size_t>(token));
+ auto ini_token = static_cast<GUI::IniToken::Type>(token);
  return ini_token == GUI::IniToken::Type::Name;
 }
 
@@ -53,7 +53,7 @@ void IniSyntaxHighlighter::rehighlight(const Palette& palette)
  span.attributes.color = style.color;
  span.attributes.bold = style.bold;
  span.is_skippable = token.m_type == IniToken::Type::Whitespace;
- span.data = reinterpret_cast<void*>(token.m_type);
+ span.data = static_cast<u64>(token.m_type);
  spans.append(span);
  }
  m_client->do_set_spans(move(spans));
@@ -64,18 +64,18 @@ void IniSyntaxHighlighter::rehighlight(const Palette& palette)
  m_client->do_update();
 }
 
-Vector<IniSyntaxHighlighter::MatchingTokenPair> IniSyntaxHighlighter::matching_token_pairs() const
+Vector<IniSyntaxHighlighter::MatchingTokenPair> IniSyntaxHighlighter::matching_token_pairs_impl() const
 {
  static Vector<MatchingTokenPair> pairs;
  if (pairs.is_empty()) {
- pairs.append({ reinterpret_cast<void*>(IniToken::Type::LeftBracket), reinterpret_cast<void*>(IniToken::Type::RightBracket) });
+ pairs.append({ static_cast<u64>(IniToken::Type::LeftBracket), static_cast<u64>(IniToken::Type::RightBracket) });
  }
  return pairs;
 }
 
-bool IniSyntaxHighlighter::token_types_equal(void* token1, void* token2) const
+bool IniSyntaxHighlighter::token_types_equal(u64 token1, u64 token2) const
 {
- return static_cast<GUI::IniToken::Type>(reinterpret_cast<size_t>(token1)) == static_cast<GUI::IniToken::Type>(reinterpret_cast<size_t>(token2));
+ return static_cast<GUI::IniToken::Type>(token1) == static_cast<GUI::IniToken::Type>(token2);
 }
 
 IniSyntaxHighlighter::~IniSyntaxHighlighter()

Userland/Libraries/LibGUI/INISyntaxHighlighter.h

@@ -15,14 +15,14 @@ class IniSyntaxHighlighter final : public Syntax::Highlighter {
  IniSyntaxHighlighter() { }
  virtual ~IniSyntaxHighlighter() override;
 
- virtual bool is_identifier(void*) const override;
+ virtual bool is_identifier(u64) const override;
 
  virtual Syntax::Language language() const override { return Syntax::Language::INI; }
  virtual void rehighlight(const Palette&) override;
 
 protected:
- virtual Vector<MatchingTokenPair> matching_token_pairs() const override;
- virtual bool token_types_equal(void*, void*) const override;
+ virtual Vector<MatchingTokenPair> matching_token_pairs_impl() const override;
+ virtual bool token_types_equal(u64, u64) const override;
 };
 
 }

Userland/Libraries/LibGUI/TextDocument.h

@@ -26,7 +26,7 @@ namespace GUI {
 struct TextDocumentSpan {
  TextRange range;
  Gfx::TextAttributes attributes;
- void* data { nullptr };
+ u64 data { 0 };
  bool is_skippable { false };
 };
 

Userland/Libraries/LibJS/SyntaxHighlighter.cpp

@@ -38,13 +38,13 @@ static Syntax::TextStyle style_for_token_type(const Gfx::Palette& palette, JS::T
  }
 }
 
-bool SyntaxHighlighter::is_identifier(void* token) const
+bool SyntaxHighlighter::is_identifier(u64 token) const
 {
- auto js_token = static_cast<JS::TokenType>(reinterpret_cast<size_t>(token));
+ auto js_token = static_cast<JS::TokenType>(static_cast<size_t>(token));
  return js_token == JS::TokenType::Identifier;
 }
 
-bool SyntaxHighlighter::is_navigatable([[maybe_unused]] void* token) const
+bool SyntaxHighlighter::is_navigatable([[maybe_unused]] u64 token) const
 {
  return false;
 }
@@ -83,7 +83,7 @@ void SyntaxHighlighter::rehighlight(const Palette& palette)
  span.attributes.color = style.color;
  span.attributes.bold = style.bold;
  span.is_skippable = is_trivia;
- span.data = reinterpret_cast<void*>(static_cast<size_t>(type));
+ span.data = static_cast<u64>(type);
  spans.append(span);
 
  dbgln_if(SYNTAX_HIGHLIGHTING_DEBUG, "{}{} @ '{}' {}:{} - {}:{}",
@@ -111,20 +111,20 @@ void SyntaxHighlighter::rehighlight(const Palette& palette)
  m_client->do_update();
 }
 
-Vector<Syntax::Highlighter::MatchingTokenPair> SyntaxHighlighter::matching_token_pairs() const
+Vector<Syntax::Highlighter::MatchingTokenPair> SyntaxHighlighter::matching_token_pairs_impl() const
 {
  static Vector<Syntax::Highlighter::MatchingTokenPair> pairs;
  if (pairs.is_empty()) {
- pairs.append({ reinterpret_cast<void*>(JS::TokenType::CurlyOpen), reinterpret_cast<void*>(JS::TokenType::CurlyClose) });
- pairs.append({ reinterpret_cast<void*>(JS::TokenType::ParenOpen), reinterpret_cast<void*>(JS::TokenType::ParenClose) });
- pairs.append({ reinterpret_cast<void*>(JS::TokenType::BracketOpen), reinterpret_cast<void*>(JS::TokenType::BracketClose) });
+ pairs.append({ static_cast<u64>(JS::TokenType::CurlyOpen), static_cast<u64>(JS::TokenType::CurlyClose) });
+ pairs.append({ static_cast<u64>(JS::TokenType::ParenOpen), static_cast<u64>(JS::TokenType::ParenClose) });
+ pairs.append({ static_cast<u64>(JS::TokenType::BracketOpen), static_cast<u64>(JS::TokenType::BracketClose) });
  }
  return pairs;
 }
 
-bool SyntaxHighlighter::token_types_equal(void* token1, void* token2) const
+bool SyntaxHighlighter::token_types_equal(u64 token1, u64 token2) const
 {
- return static_cast<JS::TokenType>(reinterpret_cast<size_t>(token1)) == static_cast<JS::TokenType>(reinterpret_cast<size_t>(token2));
+ return static_cast<JS::TokenType>(token1) == static_cast<JS::TokenType>(token2);
 }
 
 SyntaxHighlighter::~SyntaxHighlighter()

Userland/Libraries/LibJS/SyntaxHighlighter.h

@@ -15,15 +15,15 @@ class SyntaxHighlighter : public Syntax::Highlighter {
  SyntaxHighlighter() { }
  virtual ~SyntaxHighlighter() override;
 
- virtual bool is_identifier(void*) const override;
- virtual bool is_navigatable(void*) const override;
+ virtual bool is_identifier(u64) const override;
+ virtual bool is_navigatable(u64) const override;
 
  virtual Syntax::Language language() const override { return Syntax::Language::JavaScript; }
  virtual void rehighlight(const Palette&) override;
 
 protected:
- virtual Vector<MatchingTokenPair> matching_token_pairs() const override;
- virtual bool token_types_equal(void*, void*) const override;
+ virtual Vector<MatchingTokenPair> matching_token_pairs_impl() const override;
+ virtual bool token_types_equal(u64, u64) const override;
 };
 
 }

Userland/Libraries/LibSQL/SyntaxHighlighter.cpp

@@ -33,9 +33,9 @@ static Syntax::TextStyle style_for_token_type(Gfx::Palette const& palette, Token
  }
 }
 
-bool SyntaxHighlighter::is_identifier(void* token) const
+bool SyntaxHighlighter::is_identifier(u64 token) const
 {
- auto sql_token = static_cast<SQL::TokenType>(reinterpret_cast<size_t>(token));
+ auto sql_token = static_cast<SQL::TokenType>(static_cast<size_t>(token));
  return sql_token == SQL::TokenType::Identifier;
 }
 
@@ -66,7 +66,7 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
  auto style = style_for_token_type(palette, token.type());
  span.attributes.color = style.color;
  span.attributes.bold = style.bold;
- span.data = reinterpret_cast<void*>(static_cast<size_t>(token.type()));
+ span.data = static_cast<u64>(token.type());
  spans.append(span);
 
  dbgln_if(SYNTAX_HIGHLIGHTING_DEBUG, "{} @ '{}' {}:{} - {}:{}",
@@ -91,18 +91,18 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
  m_client->do_update();
 }
 
-Vector<SyntaxHighlighter::MatchingTokenPair> SyntaxHighlighter::matching_token_pairs() const
+Vector<SyntaxHighlighter::MatchingTokenPair> SyntaxHighlighter::matching_token_pairs_impl() const
 {
  static Vector<SyntaxHighlighter::MatchingTokenPair> pairs;
  if (pairs.is_empty()) {
- pairs.append({ reinterpret_cast<void*>(TokenType::ParenOpen), reinterpret_cast<void*>(TokenType::ParenClose) });
+ pairs.append({ static_cast<u64>(TokenType::ParenOpen), static_cast<u64>(TokenType::ParenClose) });
  }
  return pairs;
 }
 
-bool SyntaxHighlighter::token_types_equal(void* token1, void* token2) const
+bool SyntaxHighlighter::token_types_equal(u64 token1, u64 token2) const
 {
- return static_cast<TokenType>(reinterpret_cast<size_t>(token1)) == static_cast<TokenType>(reinterpret_cast<size_t>(token2));
+ return static_cast<TokenType>(token1) == static_cast<TokenType>(token2);
 }
 
 SyntaxHighlighter::~SyntaxHighlighter()

Userland/Libraries/LibSQL/SyntaxHighlighter.h

@@ -15,14 +15,14 @@ class SyntaxHighlighter final : public Syntax::Highlighter {
  SyntaxHighlighter() { }
  virtual ~SyntaxHighlighter() override;
 
- virtual bool is_identifier(void*) const override;
+ virtual bool is_identifier(u64) const override;
 
  virtual Syntax::Language language() const override { return Syntax::Language::SQL; }
  virtual void rehighlight(Palette const&) override;
 
 protected:
- virtual Vector<MatchingTokenPair> matching_token_pairs() const override;
- virtual bool token_types_equal(void*, void*) const override;
+ virtual Vector<MatchingTokenPair> matching_token_pairs_impl() const override;
+ virtual bool token_types_equal(u64, u64) const override;
 };
 
 }

Userland/Libraries/LibSyntax/Highlighter.cpp

@@ -23,7 +23,7 @@ void Highlighter::highlight_matching_token_pair()
  Backward,
  };
 
- auto find_span_of_type = [&](auto i, void* type, void* not_type, Direction direction) -> Optional<size_t> {
+ auto find_span_of_type = [&](auto i, u64 type, u64 not_type, Direction direction) -> Optional<size_t> {
  size_t nesting_level = 0;
  bool forward = direction == Direction::Forward;
 
@@ -130,4 +130,19 @@ void Highlighter::cursor_did_change()
  highlight_matching_token_pair();
 }
 
+Vector<Highlighter::MatchingTokenPair> Highlighter::matching_token_pairs() const
+{
+ auto own_pairs = matching_token_pairs_impl();
+ own_pairs.ensure_capacity(own_pairs.size() + m_nested_token_pairs.size());
+ for (auto& nested_pair : m_nested_token_pairs)
+ own_pairs.append(nested_pair);
+ return own_pairs;
+}
+
+void Highlighter::register_nested_token_pairs(Vector<MatchingTokenPair> pairs)
+{
+ for (auto& pair : pairs)
+ m_nested_token_pairs.set(pair);
+}
+
 }