LibSQL: Introduce a SQL library with a tokenizer

LibSQL aims to be a SQLite clone for SerenityOS. Step 1 is creating a tokenizer to lex SQL tokens. This lexer is heavily influenced by the LibJS lexer.
cubiclove · Apr 20, 2021 · 90517da · 90517da
1 parent 55b7b8e
commit 90517da
Show file tree

Hide file tree

Showing 6 changed files with 716 additions and 0 deletions.
diff --git a/Userland/Libraries/CMakeLists.txt b/Userland/Libraries/CMakeLists.txt
@@ -27,6 +27,7 @@ add_subdirectory(LibPCIDB)
 add_subdirectory(LibProtocol)
 add_subdirectory(LibPthread)
 add_subdirectory(LibRegex)
+add_subdirectory(LibSQL)
 add_subdirectory(LibSymbolClient)
 add_subdirectory(LibSyntax)
 add_subdirectory(LibSystem)

diff --git a/Userland/Libraries/LibSQL/CMakeLists.txt b/Userland/Libraries/LibSQL/CMakeLists.txt
@@ -0,0 +1,7 @@
+set(SOURCES
+ Lexer.cpp
+ Token.cpp
+)
+
+serenity_lib(LibSQL sql)
+target_link_libraries(LibSQL LibCore)
diff --git a/Userland/Libraries/LibSQL/Lexer.cpp b/Userland/Libraries/LibSQL/Lexer.cpp
@@ -0,0 +1,306 @@
+/*
+ * Copyright (c) 2021, Tim Flynn <[email protected]>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "Lexer.h"
+#include <AK/Debug.h>
+#include <ctype.h>
+
+namespace SQL {
+
+HashMap<String, TokenType> Lexer::s_keywords;
+HashMap<char, TokenType> Lexer::s_one_char_tokens;
+HashMap<String, TokenType> Lexer::s_two_char_tokens;
+
+Lexer::Lexer(StringView source)
+ : m_source(source)
+{
+ if (s_keywords.is_empty()) {
+#define __ENUMERATE_SQL_TOKEN(value, type, category) \
+ if (TokenCategory::category == TokenCategory::Keyword) \
+ s_keywords.set(value, TokenType::type);
+ ENUMERATE_SQL_TOKENS
+#undef __ENUMERATE_SQL_TOKEN
+ }
+
+ if (s_one_char_tokens.is_empty()) {
+#define __ENUMERATE_SQL_TOKEN(value, type, category) \
+ if (TokenCategory::category != TokenCategory::Keyword && StringView(value).length() == 1) \
+ s_one_char_tokens.set(value[0], TokenType::type);
+ ENUMERATE_SQL_TOKENS
+#undef __ENUMERATE_SQL_TOKEN
+ }
+
+ if (s_two_char_tokens.is_empty()) {
+#define __ENUMERATE_SQL_TOKEN(value, type, category) \
+ if (TokenCategory::category != TokenCategory::Keyword && StringView(value).length() == 2) \
+ s_two_char_tokens.set(value, TokenType::type);
+ ENUMERATE_SQL_TOKENS
+#undef __ENUMERATE_SQL_TOKEN
+ }
+
+ consume();
+}
+
+Token Lexer::next()
+{
+ bool found_invalid_comment = consume_whitespace_and_comments();
+
+ size_t value_start = m_position;
+ size_t value_start_line_number = m_line_number;
+ size_t value_start_column_number = m_line_column;
+ auto token_type = TokenType::Invalid;
+
+ if (is_eof()) {
+ token_type = found_invalid_comment ? TokenType::Invalid : TokenType::Eof;
+ } else if (is_identifier_start()) {
+ do {
+ consume();
+ } while (is_identifier_middle());
+
+ if (auto it = s_keywords.find(m_source.substring_view(value_start - 1, m_position - value_start)); it != s_keywords.end()) {
+ token_type = it->value;
+ } else {
+ token_type = TokenType::Identifier;
+ }
+ } else if (is_numeric_literal_start()) {
+ token_type = TokenType::NumericLiteral;
+ if (!consume_numeric_literal())
+ token_type = TokenType::Invalid;
+ } else {
+ bool found_two_char_token = false;
+ if (m_position < m_source.length()) {
+ if (auto it = s_two_char_tokens.find(m_source.substring_view(m_position - 1, 2)); it != s_two_char_tokens.end()) {
+ found_two_char_token = true;
+ token_type = it->value;
+ consume();
+ consume();
+ }
+ }
+
+ bool found_one_char_token = false;
+ if (!found_two_char_token) {
+ if (auto it = s_one_char_tokens.find(m_current_char); it != s_one_char_tokens.end()) {
+ found_one_char_token = true;
+ token_type = it->value;
+ consume();
+ }
+ }
+
+ if (!found_two_char_token && !found_one_char_token) {
+ token_type = TokenType::Invalid;
+ consume();
+ }
+ }
+
+ Token token(token_type, m_source.substring_view(value_start - 1, m_position - value_start), value_start_line_number, value_start_column_number);
+
+ if constexpr (SQL_DEBUG) {
+ dbgln("------------------------------");
+ dbgln("Token: {}", token.name());
+ dbgln("Value: {}", token.value());
+ dbgln("Line: {}, Column: {}", token.line_number(), token.line_column());
+ dbgln("------------------------------");
+ }
+
+ return token;
+}
+
+void Lexer::consume()
+{
+ auto did_reach_eof = [this] {
+ if (m_position != m_source.length())
+ return false;
+ m_current_char = EOF;
+ ++m_line_column;
+ ++m_position;
+ return true;
+ };
+
+ if (m_position > m_source.length())
+ return;
+
+ if (did_reach_eof())
+ return;
+
+ if (is_line_break()) {
+ ++m_line_number;
+ m_line_column = 1;
+ } else {
+ ++m_line_column;
+ }
+
+ m_current_char = m_source[m_position++];
+}
+
+bool Lexer::consume_whitespace_and_comments()
+{
+ bool found_invalid_comment = false;
+
+ while (true) {
+ if (isspace(m_current_char)) {
+ do {
+ consume();
+ } while (isspace(m_current_char));
+ } else if (is_line_comment_start()) {
+ consume();
+ do {
+ consume();
+ } while (!is_eof() && !is_line_break());
+ } else if (is_block_comment_start()) {
+ consume();
+ do {
+ consume();
+ } while (!is_eof() && !is_block_comment_end());
+ if (is_eof())
+ found_invalid_comment = true;
+ consume(); // consume *
+ if (is_eof())
+ found_invalid_comment = true;
+ consume(); // consume /
+ } else {
+ break;
+ }
+ }
+
+ return found_invalid_comment;
+}
+
+bool Lexer::consume_numeric_literal()
+{
+ // https://www.sqlite.org/draft/syntax/numeric-literal.html
+ bool is_valid_numeric_literal = true;
+
+ if (m_current_char == '0') {
+ consume();
+ if (m_current_char == '.') {
+ consume();
+ while (isdigit(m_current_char))
+ consume();
+ if (m_current_char == 'e' || m_current_char == 'E')
+ is_valid_numeric_literal = consume_exponent();
+ } else if (m_current_char == 'e' || m_current_char == 'E') {
+ is_valid_numeric_literal = consume_exponent();
+ } else if (m_current_char == 'x' || m_current_char == 'X') {
+ is_valid_numeric_literal = consume_hexadecimal_number();
+ } else if (isdigit(m_current_char)) {
+ do {
+ consume();
+ } while (isdigit(m_current_char));
+ }
+ } else {
+ do {
+ consume();
+ } while (isdigit(m_current_char));
+
+ if (m_current_char == '.') {
+ consume();
+ while (isdigit(m_current_char))
+ consume();
+ }
+ if (m_current_char == 'e' || m_current_char == 'E')
+ is_valid_numeric_literal = consume_exponent();
+ }
+
+ return is_valid_numeric_literal;
+}
+
+bool Lexer::consume_exponent()
+{
+ consume();
+ if (m_current_char == '-' || m_current_char == '+')
+ consume();
+
+ if (!isdigit(m_current_char))
+ return false;
+
+ while (isdigit(m_current_char)) {
+ consume();
+ }
+ return true;
+}
+
+bool Lexer::consume_hexadecimal_number()
+{
+ consume();
+ if (!isxdigit(m_current_char))
+ return false;
+
+ while (isxdigit(m_current_char))
+ consume();
+
+ return true;
+}
+
+bool Lexer::match(char a, char b) const
+{
+ if (m_position >= m_source.length())
+ return false;
+
+ return m_current_char == a
+ && m_source[m_position] == b;
+}
+
+bool Lexer::is_identifier_start() const
+{
+ return isalpha(m_current_char) || m_current_char == '_';
+}
+
+bool Lexer::is_identifier_middle() const
+{
+ return is_identifier_start() || isdigit(m_current_char);
+}
+
+bool Lexer::is_numeric_literal_start() const
+{
+ return isdigit(m_current_char) || (m_current_char == '.' && m_position < m_source.length() && isdigit(m_source[m_position]));
+}
+
+bool Lexer::is_line_comment_start() const
+{
+ return match('-', '-');
+}
+
+bool Lexer::is_block_comment_start() const
+{
+ return match('/', '*');
+}
+
+bool Lexer::is_block_comment_end() const
+{
+ return match('*', '/');
+}
+
+bool Lexer::is_line_break() const
+{
+ return m_current_char == '\n';
+}
+
+bool Lexer::is_eof() const
+{
+ return m_current_char == EOF;
+}
+
+}
diff --git a/Userland/Libraries/LibSQL/Lexer.h b/Userland/Libraries/LibSQL/Lexer.h
@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2021, Tim Flynn <[email protected]>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#include "Token.h"
+#include <AK/HashMap.h>
+#include <AK/String.h>
+#include <AK/StringView.h>
+
+namespace SQL {
+
+class Lexer {
+public:
+ explicit Lexer(StringView source);
+
+ Token next();
+
+private:
+ void consume();
+
+ bool consume_whitespace_and_comments();
+ bool consume_numeric_literal();
+ bool consume_exponent();
+ bool consume_hexadecimal_number();
+
+ bool match(char a, char b) const;
+ bool is_identifier_start() const;
+ bool is_identifier_middle() const;
+ bool is_numeric_literal_start() const;
+ bool is_line_comment_start() const;
+ bool is_block_comment_start() const;
+ bool is_block_comment_end() const;
+ bool is_line_break() const;
+ bool is_eof() const;
+
+ static HashMap<String, TokenType> s_keywords;
+ static HashMap<char, TokenType> s_one_char_tokens;
+ static HashMap<String, TokenType> s_two_char_tokens;
+
+ StringView m_source;
+ size_t m_line_number { 1 };
+ size_t m_line_column { 0 };
+ char m_current_char { 0 };
+ size_t m_position { 0 };
+};
+
+}