From 8b717f5253ecf9396d55f6cfa73176f7a4565916 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Sun, 22 Nov 2020 21:25:12 -1000 Subject: [PATCH] Full stack mysql test environment Created mysql environment heavily insipred (copy/pasted) by the postgres module. MysqlModule.scala: performs the conversion of zio-sql functions into mysql compatible sql. Currently a copy/paste & rename of the postgres module. FunctionDefSpec.scala: module for testing mysql functions (lower, sin, etc.) MysqlModuleTest.scala: runs mysql tests, table selects, limits, offset, etc. MysqlRunnableSpec.scala: ShopSchema.scala: defines schema (loaded in test container definition) --- build.sbt | 38 +-- .../scala/zio/sql/mysql/MysqlModule.scala | 225 ++++++++++++++++++ mysql/src/test/resources/shop_schema.sql | 12 - .../scala/zio/sql/mysql/FunctionDefSpec.scala | 39 +++ .../scala/zio/sql/mysql/MysqlModuleTest.scala | 96 ++++++++ .../zio/sql/mysql/MysqlRunnableSpec.scala | 33 +++ .../test/scala/zio/sql/mysql/ShopSchema.scala | 41 ++++ 7 files changed, 453 insertions(+), 31 deletions(-) create mode 100644 mysql/src/main/scala/zio/sql/mysql/MysqlModule.scala create mode 100644 mysql/src/test/scala/zio/sql/mysql/FunctionDefSpec.scala create mode 100644 mysql/src/test/scala/zio/sql/mysql/MysqlModuleTest.scala create mode 100644 mysql/src/test/scala/zio/sql/mysql/MysqlRunnableSpec.scala create mode 100644 mysql/src/test/scala/zio/sql/mysql/ShopSchema.scala diff --git a/build.sbt b/build.sbt index 236e54733..9e9d919f3 100644 --- a/build.sbt +++ b/build.sbt @@ -23,8 +23,8 @@ inThisBuild( addCommandAlias("fmt", "all scalafmtSbt scalafmt test:scalafmt") addCommandAlias("check", "all scalafmtSbtCheck scalafmtCheck test:scalafmtCheck") -val zioVersion = "1.0.3" -val testcontainersVersion = "1.15.0" +val zioVersion = "1.0.3" +val testcontainersVersion = "1.15.0" val testcontainersScalaVersion = "1.0.0-alpha1" lazy val startPostgres = taskKey[Unit]("Start up Postgres") @@ -152,15 +152,15 @@ lazy val mysql = project .settings(buildInfoSettings("zio.sql.mysql")) .settings( libraryDependencies ++= Seq( - "dev.zio" %% "zio" % zioVersion, - "dev.zio" %% "zio-test" % zioVersion % "test", - "dev.zio" %% "zio-test-sbt" % zioVersion % "test", - "mysql" % "mysql-connector-java" % "8.0.22", - "org.testcontainers" % "testcontainers" % testcontainersVersion % Test, - "org.testcontainers" % "database-commons" % testcontainersVersion % Test, - "org.testcontainers" % "jdbc" % testcontainersVersion % Test, - "org.testcontainers" % "mysql" % testcontainersVersion % Test, - "com.dimafeng" %% "testcontainers-scala-mysql" % testcontainersScalaVersion % Test, + "dev.zio" %% "zio" % zioVersion, + "dev.zio" %% "zio-test" % zioVersion % "test", + "dev.zio" %% "zio-test-sbt" % zioVersion % "test", + "mysql" % "mysql-connector-java" % "8.0.22", + "org.testcontainers" % "testcontainers" % testcontainersVersion % Test, + "org.testcontainers" % "database-commons" % testcontainersVersion % Test, + "org.testcontainers" % "jdbc" % testcontainersVersion % Test, + "org.testcontainers" % "mysql" % testcontainersVersion % Test, + "com.dimafeng" %% "testcontainers-scala-mysql" % testcontainersScalaVersion % Test ) ) .settings(testFrameworks += new TestFramework("zio.test.sbt.ZTestFramework")) @@ -189,14 +189,14 @@ lazy val postgres = project .settings( libraryDependencies ++= Seq( "dev.zio" %% "zio" % zioVersion, - "dev.zio" %% "zio-test" % zioVersion % Test, - "dev.zio" %% "zio-test-sbt" % zioVersion % Test, - "org.testcontainers" % "testcontainers" % testcontainersVersion % Test, - "org.testcontainers" % "database-commons" % testcontainersVersion % Test, - "org.testcontainers" % "postgresql" % testcontainersVersion % Test, - "org.testcontainers" % "jdbc" % testcontainersVersion % Test, - "org.postgresql" % "postgresql" % "42.2.18" % Test, - "com.dimafeng" %% "testcontainers-scala-postgresql" % testcontainersScalaVersion % Test, + "dev.zio" %% "zio-test" % zioVersion % Test, + "dev.zio" %% "zio-test-sbt" % zioVersion % Test, + "org.testcontainers" % "testcontainers" % testcontainersVersion % Test, + "org.testcontainers" % "database-commons" % testcontainersVersion % Test, + "org.testcontainers" % "postgresql" % testcontainersVersion % Test, + "org.testcontainers" % "jdbc" % testcontainersVersion % Test, + "org.postgresql" % "postgresql" % "42.2.18" % Test, + "com.dimafeng" %% "testcontainers-scala-postgresql" % testcontainersScalaVersion % Test ) ) .settings(testFrameworks += new TestFramework("zio.test.sbt.ZTestFramework")) diff --git a/mysql/src/main/scala/zio/sql/mysql/MysqlModule.scala b/mysql/src/main/scala/zio/sql/mysql/MysqlModule.scala new file mode 100644 index 000000000..712bcd81e --- /dev/null +++ b/mysql/src/main/scala/zio/sql/mysql/MysqlModule.scala @@ -0,0 +1,225 @@ +package zio.sql.mysql + +import zio.sql.Jdbc + +trait MysqlModule extends Jdbc { self => + object MysqlFunctionDef {} + + override def renderRead(read: self.Read[_]): String = { + val builder = new StringBuilder + + def buildExpr[A, B](expr: self.Expr[_, A, B]): Unit = expr match { + case Expr.Source(tableName, column) => + val _ = builder.append(tableName).append(".").append(column.name) + case Expr.Unary(base, op) => + val _ = builder.append(" ").append(op.symbol) + buildExpr(base) + case Expr.Property(base, op) => + buildExpr(base) + val _ = builder.append(" ").append(op.symbol) + case Expr.Binary(left, right, op) => + buildExpr(left) + builder.append(" ").append(op.symbol).append(" ") + buildExpr(right) + case Expr.Relational(left, right, op) => + buildExpr(left) + builder.append(" ").append(op.symbol).append(" ") + buildExpr(right) + case Expr.In(value, set) => + buildExpr(value) + buildReadString(set) + case Expr.Literal(value) => + val _ = builder.append(value.toString) //todo fix escaping + case Expr.AggregationCall(param, aggregation) => + builder.append(aggregation.name.name) + builder.append("(") + buildExpr(param) + val _ = builder.append(")") + case Expr.FunctionCall0(function) => + val _ = builder.append(function.name.name) + case Expr.FunctionCall1(param, function) => + builder.append(function.name.name) + builder.append("(") + buildExpr(param) + val _ = builder.append(")") + case Expr.FunctionCall2(param1, param2, function) => + builder.append(function.name.name) + builder.append("(") + buildExpr(param1) + builder.append(",") + buildExpr(param2) + val _ = builder.append(")") + case Expr.FunctionCall3(param1, param2, param3, function) => + builder.append(function.name.name) + builder.append("(") + buildExpr(param1) + builder.append(",") + buildExpr(param2) + builder.append(",") + buildExpr(param3) + val _ = builder.append(")") + case Expr.FunctionCall4(param1, param2, param3, param4, function) => + builder.append(function.name.name) + builder.append("(") + buildExpr(param1) + builder.append(",") + buildExpr(param2) + builder.append(",") + buildExpr(param3) + builder.append(",") + buildExpr(param4) + val _ = builder.append(")") + } + + def buildReadString[A <: SelectionSet[_]](read: self.Read[_]): Unit = + read match { + case read0 @ Read.Select(_, _, _, _, _, _, _, _) => + object Dummy { + type F + type A + type B <: SelectionSet[A] + } + val read = read0.asInstanceOf[Read.Select[Dummy.F, Dummy.A, Dummy.B]] + import read._ + + builder.append("SELECT ") + buildSelection(selection.value) + builder.append(" FROM ") + buildTable(table) + whereExpr match { + case Expr.Literal(true) => () + case _ => + builder.append(" WHERE ") + buildExpr(whereExpr) + } + groupBy match { + case _ :: _ => + builder.append(" GROUP BY ") + buildExprList(groupBy) + + havingExpr match { + case Expr.Literal(true) => () + case _ => + builder.append(" HAVING ") + buildExpr(havingExpr) + } + case Nil => () + } + orderBy match { + case _ :: _ => + builder.append(" ORDER BY ") + buildOrderingList(orderBy) + case Nil => () + } + limit match { + case Some(limit) => + builder.append(" LIMIT ").append(limit) + case None => () + } + offset match { + case Some(offset) => + val _ = builder.append(" OFFSET ").append(offset) + case None => () + } + + case Read.Union(left, right, distinct) => + buildReadString(left) + builder.append(" UNION ") + if (!distinct) builder.append("ALL ") + buildReadString(right) + + case Read.Literal(values) => + val _ = builder.append(" (").append(values.mkString(",")).append(") ") //todo fix needs escaping + } + + def buildExprList(expr: List[Expr[_, _, _]]): Unit = + expr match { + case head :: tail => + buildExpr(head) + tail match { + case _ :: _ => + builder.append(", ") + buildExprList(tail) + case Nil => () + } + case Nil => () + } + def buildOrderingList(expr: List[Ordering[Expr[_, _, _]]]): Unit = + expr match { + case head :: tail => + head match { + case Ordering.Asc(value) => buildExpr(value) + case Ordering.Desc(value) => + buildExpr(value) + builder.append(" DESC") + } + tail match { + case _ :: _ => + builder.append(", ") + buildOrderingList(tail) + case Nil => () + } + case Nil => () + } + + def buildSelection[A](selectionSet: SelectionSet[A]): Unit = + selectionSet match { + case cons0 @ SelectionSet.Cons(_, _) => + object Dummy { + type Source + type A + type B <: SelectionSet[Source] + } + val cons = cons0.asInstanceOf[SelectionSet.Cons[Dummy.Source, Dummy.A, Dummy.B]] + import cons._ + buildColumnSelection(head) + if (tail != SelectionSet.Empty) { + builder.append(", ") + buildSelection(tail) + } + case SelectionSet.Empty => () + } + + def buildColumnSelection[A, B](columnSelection: ColumnSelection[A, B]): Unit = + columnSelection match { + case ColumnSelection.Constant(value, name) => + builder.append(value.toString) //todo fix escaping + name match { + case Some(name) => + val _ = builder.append(" AS ").append(name) + case None => () + } + case ColumnSelection.Computed(expr, name) => + buildExpr(expr) + name match { + case Some(name) => + Expr.exprName(expr) match { + case Some(sourceName) if name != sourceName => + val _ = builder.append(" AS ").append(name) + case _ => () + } + case _ => () //todo what do we do if we don't have a name? + } + } + def buildTable(table: Table): Unit = + table match { + //The outer reference in this type test cannot be checked at run time?! + case sourceTable: self.Table.Source => + val _ = builder.append(sourceTable.name) + case Table.Joined(joinType, left, right, on) => + buildTable(left) + builder.append(joinType match { + case JoinType.Inner => " INNER JOIN " + case JoinType.LeftOuter => " LEFT JOIN " + case JoinType.RightOuter => " RIGHT JOIN " + case JoinType.FullOuter => " OUTER JOIN " + }) + buildTable(right) + builder.append(" ON ") + buildExpr(on) + val _ = builder.append(" ") + } + buildReadString(read) + builder.toString() + } +} diff --git a/mysql/src/test/resources/shop_schema.sql b/mysql/src/test/resources/shop_schema.sql index db62c3669..8f3e9a6ba 100644 --- a/mysql/src/test/resources/shop_schema.sql +++ b/mysql/src/test/resources/shop_schema.sql @@ -1,9 +1,3 @@ -create table simple -( - id int not null primary key, - message varchar(255) not null -); - create table customers ( id varchar(36) not null primary key, @@ -28,12 +22,6 @@ create table products image_url varchar(255) ); -insert into simple - (id, message) -values - (1, "Test message"), - (2, "Another test"); - insert into customers (id, first_name, last_name, verified, dob) values diff --git a/mysql/src/test/scala/zio/sql/mysql/FunctionDefSpec.scala b/mysql/src/test/scala/zio/sql/mysql/FunctionDefSpec.scala new file mode 100644 index 000000000..88a7eef44 --- /dev/null +++ b/mysql/src/test/scala/zio/sql/mysql/FunctionDefSpec.scala @@ -0,0 +1,39 @@ +package zio.sql.mysql + +import zio.Cause +import zio.test.Assertion._ +import zio.test._ + +object FunctionDefSpec extends MysqlRunnableSpec with ShopSchema { + import this.Customers._ + import this.FunctionDef._ + + val spec = suite("Mysql FunctionDef")( + testM("lower") { + val query = select(Lower("first_name")) from customers limit (1) + + val expected = "ronald" + + val testResult = execute(query).to[String, String](identity) + + val assertion = for { + r <- testResult.runCollect + } yield assert(r.head)(equalTo(expected)) + + assertion.mapErrorCause(cause => Cause.stackless(cause.untraced)) + }, + testM("sin") { + val query = select(Sin(1.0)) from customers + + val expected = 0.8414709848078965 + + val testResult = execute(query).to[Double, Double](identity) + + val assertion = for { + r <- testResult.runCollect + } yield assert(r.head)(equalTo(expected)) + + assertion.mapErrorCause(cause => Cause.stackless(cause.untraced)) + } + ) +} diff --git a/mysql/src/test/scala/zio/sql/mysql/MysqlModuleTest.scala b/mysql/src/test/scala/zio/sql/mysql/MysqlModuleTest.scala new file mode 100644 index 000000000..2151ce9fa --- /dev/null +++ b/mysql/src/test/scala/zio/sql/mysql/MysqlModuleTest.scala @@ -0,0 +1,96 @@ +package zio.sql.mysql + +import java.time.LocalDate +import java.util.UUID + +import zio.Cause +import zio.test.Assertion._ +import zio.test._ + +object MysqlModuleTest extends MysqlRunnableSpec with ShopSchema { + + import Customers._ + + val spec = suite("Mysql module")( + testM("can select from single table") { + case class Customer(id: UUID, fname: String, lname: String, dateOfBirth: LocalDate) + + val query = select(customerId ++ fName ++ lName ++ dob) from customers + + println(renderRead(query)) + + val expected = + Seq( + Customer( + UUID.fromString("60b01fc9-c902-4468-8d49-3c0f989def37"), + "Ronald", + "Russell", + LocalDate.parse("1983-01-05") + ), + Customer( + UUID.fromString("f76c9ace-be07-4bf3-bd4c-4a9c62882e64"), + "Terrence", + "Noel", + LocalDate.parse("1999-11-02") + ), + Customer( + UUID.fromString("784426a5-b90a-4759-afbb-571b7a0ba35e"), + "Mila", + "Paterso", + LocalDate.parse("1990-11-16") + ), + Customer( + UUID.fromString("df8215a2-d5fd-4c6c-9984-801a1b3a2a0b"), + "Alana", + "Murray", + LocalDate.parse("1995-11-12") + ), + Customer( + UUID.fromString("636ae137-5b1a-4c8c-b11f-c47c624d9cdc"), + "Jose", + "Wiggins", + LocalDate.parse("1987-03-23") + ) + ) + + val testResult = execute(query) + .to[UUID, String, String, LocalDate, Customer] { case row => + Customer(row._1, row._2, row._3, row._4) + } + + val assertion = for { + r <- testResult.runCollect + } yield assert(r)(hasSameElementsDistinct(expected)) + + assertion.mapErrorCause(cause => Cause.stackless(cause.untraced)) + }, + testM("Can select from single table with limit, offset and order by") { + case class Customer(id: UUID, fname: String, lname: String, dateOfBirth: LocalDate) + + val query = (select(customerId ++ fName ++ lName ++ dob) from customers).limit(1).offset(1).orderBy(fName) + + println(renderRead(query)) + + val expected = + Seq( + Customer( + UUID.fromString("636ae137-5b1a-4c8c-b11f-c47c624d9cdc"), + "Jose", + "Wiggins", + LocalDate.parse("1987-03-23") + ) + ) + + val testResult = execute(query) + .to[UUID, String, String, LocalDate, Customer] { case row => + Customer(row._1, row._2, row._3, row._4) + } + + val assertion = for { + r <- testResult.runCollect + } yield assert(r)(hasSameElementsDistinct(expected)) + + assertion.mapErrorCause(cause => Cause.stackless(cause.untraced)) + } + ) +} diff --git a/mysql/src/test/scala/zio/sql/mysql/MysqlRunnableSpec.scala b/mysql/src/test/scala/zio/sql/mysql/MysqlRunnableSpec.scala new file mode 100644 index 000000000..a6d04dd95 --- /dev/null +++ b/mysql/src/test/scala/zio/sql/mysql/MysqlRunnableSpec.scala @@ -0,0 +1,33 @@ +package zio.sql.mysql + +import java.util.Properties + +import zio.blocking.Blocking +import zio.sql.TestContainer +import zio.sql.postgresql.JdbcRunnableSpec +import zio.test.environment.TestEnvironment +import zio.{ Has, ZEnv, ZLayer } + +trait MysqlRunnableSpec extends JdbcRunnableSpec with MysqlModule { + + private def connProperties(user: String, password: String): Properties = { + val props = new Properties + props.setProperty("user", user) + props.setProperty("password", password) + props + } + + private val executorLayer = { + val poolConfigLayer = TestContainer + .mysql("mysql:8") + .map(a => Has(ConnectionPool.Config(a.get.jdbcUrl, connProperties(a.get.username, a.get.password)))) + + val connectionPoolLayer = Blocking.live >+> poolConfigLayer >>> ConnectionPool.live + + (Blocking.live ++ connectionPoolLayer >>> ReadExecutor.live).orDie + } + + override val jdbcTestEnvironment: ZLayer[ZEnv, Nothing, TestEnvironment with ReadExecutor] = + TestEnvironment.live ++ executorLayer + +} diff --git a/mysql/src/test/scala/zio/sql/mysql/ShopSchema.scala b/mysql/src/test/scala/zio/sql/mysql/ShopSchema.scala new file mode 100644 index 000000000..b22b2378d --- /dev/null +++ b/mysql/src/test/scala/zio/sql/mysql/ShopSchema.scala @@ -0,0 +1,41 @@ +package zio.sql.mysql + +import zio.sql.Jdbc + +trait ShopSchema extends Jdbc { self => + import self.ColumnSet._ + + object Customers { + val customers = + (uuid("id") ++ localDate("dob") ++ string("first_name") ++ string("last_name") ++ boolean("verified")) + .table("customers") + + val customerId :*: dob :*: fName :*: lName :*: verified :*: _ = customers.columns + } + object Orders { + val orders = (uuid("id") ++ uuid("customer_id") ++ localDate("order_date")).table("orders") + + val orderId :*: fkCustomerId :*: orderDate :*: _ = orders.columns + } + object Products { + val products = + (int("id") ++ string("name") ++ string("description") ++ string("image_url")).table("products") + + val productId :*: description :*: imageURL :*: _ = products.columns + } + object ProductPrices { + val productPrices = + (int("product_id") ++ offsetDateTime("effective") ++ bigDecimal("price")).table("product_prices") + + val fkProductId :*: effective :*: price :*: _ = productPrices.columns + } + object OrderDetails { + val orderDetails = + (int("order_id") ++ int("product_id") ++ double("quantity") ++ double("unit_price")) + .table( + "order_details" + ) //todo fix #3 quantity should be int, unit price should be bigDecimal, numeric operators only support double ATM. + + val fkOrderId :*: fkProductId :*: quantity :*: unitPrice :*: _ = orderDetails.columns + } +}