+Circe has more specific types than `JNumber`, like `JLong`, `JDouble` and other, +but all of them are private to Circe itself, so we can not use them, not even for matching types during conversions. +In any case, even if we were able to use them, when parsing a JSON string (probably most of the cases) +we can not detect the specific types +
+ +```scala +import io.Circe.Json +import org.datatools.bigdatatypes.circe.CirceTypeConversion.* +import org.datatools.bigdatatypes.formats.Formats.implicitDefaultFormats +import org.datatools.bigdatatypes.spark.SqlInstanceToSpark.InstanceSyntax +import org.datatools.bigdatatypes.bigquery.SqlInstanceToBigQuery.* + + val circeJson: Json = Json.fromFields(List( + ("id", Json.fromString("test")), + ("foo", Json.fromString("test")), + ("bar", Json.fromInt(1)) + )) + + val sparkSchema: StructType = circeJson.asSparkSchema + val bqSchema: Schema = circeJson.asBigQuery.schema +``` + +Or if you do it from a JSON parsed using Circe: +```scala +import io.circe._, io.circe.parser._ + +import org.datatools.bigdatatypes.circe.CirceTypeConversion.* +import org.datatools.bigdatatypes.formats.Formats.implicitDefaultFormats +import org.datatools.bigdatatypes.spark.SqlInstanceToSpark.InstanceSyntax +import org.datatools.bigdatatypes.bigquery.SqlInstanceToBigQuery.* + +val rawJson: String = """ +{ + "foo": "bar", + "baz": 123, + "list of stuff": [ 4, 5, 6 ] +} +""" +val parseResult = parse(rawJson) +// parseResult: Either[ParsingFailure, Json] +val sparkSchema = parseResult.map(j => j.asSparkSchema) +// sparkSchema: Either[ParsingFailure, StructType] +val bqSchema = parseResult.map(j => j.asBigQuery.schema) +// bqSchema: Either[ParsingFailure, Schema] +``` + +--- diff --git a/docs/intro.md b/docs/intro.md index 909a1853..fbccb59e 100644 --- a/docs/intro.md +++ b/docs/intro.md @@ -33,11 +33,12 @@ or a BigQuery table into a Cassandra table without having code that relates thos ### Available conversions: -| From / To |Scala Types |BigQuery |Spark |Cassandra | -|------------|:----------------:|:----------------:|:----------------:|:----------------:| -|Scala Types | - |:white_check_mark:|:white_check_mark:|:white_check_mark:| -|BigQuery | | - |:white_check_mark:|:white_check_mark:| -|Spark | |:white_check_mark:| - |:white_check_mark:| -|Cassandra | |:white_check_mark:|:white_check_mark:| - | +| From / To |Scala Types |BigQuery |Spark |Cassandra | Circe (JSON) | +|--------------|:----------------:|:----------------:|:----------------:|:----------------:|:------------:| +| Scala Types | - |:white_check_mark:|:white_check_mark:|:white_check_mark:| | +| BigQuery | | - |:white_check_mark:|:white_check_mark:| | +| Spark | |:white_check_mark:| - |:white_check_mark:| | +| Cassandra | |:white_check_mark:|:white_check_mark:| - | | +| Circe (JSON) | |:white_check_mark:|:white_check_mark:|:white_check_mark:| | diff --git a/examples/src/test/scala/org/datatools/bigdatatypes/CassandraToOthers.scala b/examples/src/test/scala/org/datatools/bigdatatypes/CassandraToOthers.scala index 474c881a..72b628a9 100644 --- a/examples/src/test/scala/org/datatools/bigdatatypes/CassandraToOthers.scala +++ b/examples/src/test/scala/org/datatools/bigdatatypes/CassandraToOthers.scala @@ -21,14 +21,18 @@ class CassandraToOthers extends UnitSpec { .withColumn("foo", DataTypes.TEXT) .withColumn("bar", DataTypes.INT) + val fields: List[Field] = List( + Field.newBuilder("id", StandardSQLTypeName.STRING).setMode(Mode.REQUIRED).build(), + Field.newBuilder("foo", StandardSQLTypeName.STRING).setMode(Mode.REQUIRED).build(), + Field.newBuilder("bar", StandardSQLTypeName.INT64).setMode(Mode.REQUIRED).build() + ) + val bqSchema: Schema = Schema.of(toJava(fields)) + "Cassandra table" should "be converted into BigQuery Schema" in { - val fields = List( - Field.newBuilder("id", StandardSQLTypeName.STRING).setMode(Mode.REQUIRED).build(), - Field.newBuilder("foo", StandardSQLTypeName.STRING).setMode(Mode.REQUIRED).build(), - Field.newBuilder("bar", StandardSQLTypeName.INT64).setMode(Mode.REQUIRED).build() - ) - val bqSchema = Schema.of(toJava(fields)) - SqlInstanceToBigQuery[CreateTable] + SqlInstanceToBigQuery[CreateTable].bigQueryFields(cassandraTable).schema shouldBe bqSchema + } + + it should "be converted into BigQuery Schema using extension method" in { cassandraTable.asBigQuery.schema shouldBe bqSchema } } diff --git a/examples/src/test/scala/org/datatools/bigdatatypes/CirceToOthers.scala b/examples/src/test/scala/org/datatools/bigdatatypes/CirceToOthers.scala new file mode 100644 index 00000000..89279da8 --- /dev/null +++ b/examples/src/test/scala/org/datatools/bigdatatypes/CirceToOthers.scala @@ -0,0 +1,50 @@ +package org.datatools.bigdatatypes + +import com.google.cloud.bigquery.Field.Mode +import com.google.cloud.bigquery.{Field, Schema, StandardSQLTypeName} +import io.circe.Json +import org.datatools.bigdatatypes.bigquery.JavaConverters.toJava +import org.datatools.bigdatatypes.bigquery.SqlInstanceToBigQuery.{InstanceSchemaSyntax, InstanceSyntax} +import org.datatools.bigdatatypes.cassandra.CassandraTables +import org.datatools.bigdatatypes.cassandra.CassandraTables.AsCassandraInstanceSyntax +import org.datatools.bigdatatypes.circe.CirceTypeConversion.circeJsonType +import org.datatools.bigdatatypes.formats.Formats.implicitDefaultFormats + +class CirceToOthers extends UnitSpec { + + behavior of "Circe types to Cassandra" + + val circeJson: Json = CirceTestTypes.basicTypes + + "Json from Circe" should "be converted into Cassandra Table" in { + CassandraTables.table[Json](CirceTestTypes.basicTypes, "testTable", "myLong").toString shouldBe + "CREATE TABLE testtable (myint decimal,mylong decimal PRIMARY KEY,myfloat decimal,mydouble decimal,mydecimal decimal,myboolean boolean,mystring text)" + } + + it should "be converted into Cassandra Table using extension method" in { + CirceTestTypes.basicTypes.asCassandra("testTable", "myLong").toString shouldBe + "CREATE TABLE testtable (myint decimal,mylong decimal PRIMARY KEY,myfloat decimal,mydouble decimal,mydecimal decimal,myboolean boolean,mystring text)" + } + + behavior of "Circe to BigQuery" + + val fields: List[Field] = List( + Field.newBuilder("id", StandardSQLTypeName.STRING).setMode(Mode.REQUIRED).build(), + Field.newBuilder("foo", StandardSQLTypeName.STRING).setMode(Mode.REQUIRED).build(), + Field.newBuilder("bar", StandardSQLTypeName.NUMERIC).setMode(Mode.REQUIRED).build() + ) + val bqSchema: Schema = Schema.of(toJava(fields)) + + val circe: Json = Json.fromFields( + List( + ("id", Json.fromString("test")), + ("foo", Json.fromString("test")), + ("bar", Json.fromInt(1)) + ) + ) + + it should "be converted into BigQuery Schema" in { + circe.asBigQuery.schema shouldBe bqSchema + } + +} diff --git a/examples/src/test/scala_2/bigdatatypes/CassandraToOthers.scala b/examples/src/test/scala_2/bigdatatypes/CassandraToOthers.scala index d5463078..e4692915 100644 --- a/examples/src/test/scala_2/bigdatatypes/CassandraToOthers.scala +++ b/examples/src/test/scala_2/bigdatatypes/CassandraToOthers.scala @@ -21,7 +21,6 @@ class CassandraToOthers extends UnitSpec { .withColumn("bar", DataTypes.INT) "Cassandra table" should "be converted into Spark Schema" in { - // val sparkSchema: StructType = myDataFrame.schema val sparkSchema: StructType = StructType( List( StructField("id", StringType, nullable = false), diff --git a/examples/src/test/scala_2/bigdatatypes/CirceToOthers.scala b/examples/src/test/scala_2/bigdatatypes/CirceToOthers.scala new file mode 100644 index 00000000..bcba706e --- /dev/null +++ b/examples/src/test/scala_2/bigdatatypes/CirceToOthers.scala @@ -0,0 +1,39 @@ +package bigdatatypes + +import io.circe.Json +import org.apache.spark.sql.types.{DataTypes, StringType, StructField, StructType} +import org.datatools.bigdatatypes.UnitSpec +import org.datatools.bigdatatypes.circe.CirceTypeConversion.circeJsonType +import org.datatools.bigdatatypes.formats.Formats.implicitDefaultFormats +import org.datatools.bigdatatypes.spark.SparkSchemas +import org.datatools.bigdatatypes.spark.SqlInstanceToSpark.InstanceSyntax + +class CirceToOthers extends UnitSpec { + + behavior of "Circe to Spark types" + + val sparkSchema: StructType = StructType( + List( + StructField("id", StringType, nullable = false), + StructField("foo", StringType, nullable = false), + StructField("bar", DataTypes.createDecimalType, nullable = false) + ) + ) + + val circe: Json = Json.fromFields( + List( + ("id", Json.fromString("test")), + ("foo", Json.fromString("test")), + ("bar", Json.fromInt(1)) + ) + ) + + "Circe Json" should "be converted into Spark Schema" in { + SparkSchemas.schema[Json](circe) shouldBe sparkSchema + } + + it should "be converted into Spark Schema using extension method" in { + circe.asSparkSchema shouldBe sparkSchema + } + +} diff --git a/jsoncirce/src/main/scala/org/datatools/bigdatatypes/circe/CirceTypeConversion.scala b/jsoncirce/src/main/scala/org/datatools/bigdatatypes/circe/CirceTypeConversion.scala new file mode 100644 index 00000000..600d1c70 --- /dev/null +++ b/jsoncirce/src/main/scala/org/datatools/bigdatatypes/circe/CirceTypeConversion.scala @@ -0,0 +1,50 @@ +package org.datatools.bigdatatypes.circe + +import io.circe.Json +import org.datatools.bigdatatypes.basictypes.SqlType.* +import org.datatools.bigdatatypes.basictypes.SqlTypeMode.{Repeated, Required} +import org.datatools.bigdatatypes.basictypes.{SqlType, SqlTypeMode} +import org.datatools.bigdatatypes.conversions.SqlInstanceConversion + +import scala.annotation.tailrec + +object CirceTypeConversion { + + /** Circe does not have an implementation of SqlTypeConversion due to its private API. + * We can not detect if a field is JDouble, JLong and so on because they are private, + * so the only way to detect some kind of types is based on their methods `.isNumber`, `isString` and so on + * which are only available if we have an instance of [[Json]] + */ + + /** Implementation of SqlInstanceConversion Type Class */ + implicit val circeJsonType: SqlInstanceConversion[Json] = (value: Json) => convertCirceType(value) + + @tailrec + def convertCirceType(j: Json, repeated: Boolean = false): SqlType = + j match { + case v if v.isArray => convertCirceType(v.asArray.get.apply(0), repeated = true) + case v if v.isNumber => SqlDecimal(isRepeated(repeated)) + case v if v.isString => SqlString(isRepeated(repeated)) + case v if v.isBoolean => SqlBool(isRepeated(repeated)) + case v if v.isObject => + val pairs = v.asObject.get.keys zip v.asObject.get.values + SqlStruct(loopStructs(pairs), isRepeated(repeated)) + } + + /** For recursion, loops over all items in an object + */ + private def loopStructs(l: Iterable[(String, Json)]): List[(String, SqlType)] = + l.map(x => x._1 -> convertCirceType(x._2)).toList + + /** From Boolean to Repeated or Required Mode + */ + private def isRepeated(repeated: Boolean): SqlTypeMode = if (repeated) Repeated else Required + + /** Extension method. Enables val myInstance: Json -> myInstance.asSqlType + * @param value in a Json from Circe + */ + implicit class StructTypeSyntax(value: Json) { + def asSqlType: SqlType = SqlInstanceConversion[Json].getType(value) + } + +} diff --git a/jsoncirce/src/test/scala/org/datatools/bigdatatypes/CirceTestTypes.scala b/jsoncirce/src/test/scala/org/datatools/bigdatatypes/CirceTestTypes.scala new file mode 100644 index 00000000..8edc45a0 --- /dev/null +++ b/jsoncirce/src/test/scala/org/datatools/bigdatatypes/CirceTestTypes.scala @@ -0,0 +1,61 @@ +package org.datatools.bigdatatypes + +import io.circe.Json + +/** Test types from [[TestTypes]] converted to Circe. They can be used in multiple tests + */ +object CirceTestTypes { + + val basicFields: Seq[(String, Json)] = + List( + ("myInt", Json.fromInt(1)), + ("myLong", Json.fromLong(1)), + ("myFloat", Json.fromFloat(1).get), + ("myDouble", Json.fromDouble(1).get), + ("myDecimal", Json.fromBigDecimal(1)), + ("myBoolean", Json.fromBoolean(true)), + ("myString", Json.fromString("")) + ) + + val basicTypes: Json = Json.fromFields(basicFields) + + val basicWithList: Json = Json.fromFields( + List( + ("myInt", Json.fromInt(1)), + ( + "myList", + Json.fromValues( + List( + Json.fromInt(1) + ) + ) + ) + ) + ) + + val basicNested: Json = Json.fromFields( + List( + ("myInt", Json.fromInt(1)), + ("myStruct", Json.fromFields(basicFields)) + ) + ) + + val basicNestedWithList: Json = Json.fromFields( + List( + ( + "matrix", + Json.fromValues( + List( + Json.fromFields( + List( + ("x", Json.fromInt(1)), + ("y", Json.fromInt(1)) + ) + ) + ) + ) + ) + ) + ) + +} diff --git a/jsoncirce/src/test/scala/org/datatools/bigdatatypes/SqlTestTypes.scala b/jsoncirce/src/test/scala/org/datatools/bigdatatypes/SqlTestTypes.scala new file mode 100644 index 00000000..f097cb69 --- /dev/null +++ b/jsoncirce/src/test/scala/org/datatools/bigdatatypes/SqlTestTypes.scala @@ -0,0 +1,82 @@ +package org.datatools.bigdatatypes + +import org.datatools.bigdatatypes.basictypes.* +import org.datatools.bigdatatypes.basictypes.SqlType.* +import org.datatools.bigdatatypes.basictypes.SqlTypeMode.* + +import java.sql.{Date, Timestamp} + +/** Case Classes and their SqlType representations + * This should be used to test SqlTypeConversion and all reverse conversions from other modules + */ +object SqlTestTypes { + + /** Used for case classes, nested or others */ + val basicFields: List[(String, SqlType)] = + List( + ("myInt", SqlDecimal(Required)), + ("myLong", SqlDecimal(Required)), + ("myFloat", SqlDecimal(Required)), + ("myDouble", SqlDecimal(Required)), + ("myDecimal", SqlDecimal(Required)), + ("myBoolean", SqlBool(Required)), + ("myString", SqlString(Required)) + ) + + val basicOption: SqlStruct = SqlStruct( + List( + ("myString", SqlString(Required)), + ("myOptionalString", SqlString(Nullable)) + ) + ) + + val basicTypes: SqlStruct = SqlStruct(basicFields) + + val basicOptionTypes: SqlStruct = SqlStruct( + List( + ("myInt", SqlDecimal(Nullable)), + ("myLong", SqlDecimal(Nullable)), + ("myFloat", SqlDecimal(Nullable)), + ("myDouble", SqlDecimal(Nullable)), + ("myDecimal", SqlDecimal(Nullable)), + ("myBoolean", SqlBool(Nullable)), + ("myString", SqlString(Nullable)) + ) + ) + + val basicWithList: SqlStruct = SqlStruct( + List( + ("myInt", SqlDecimal(Required)), + ("myList", SqlDecimal(Repeated)) + ) + ) + + val basicNested: SqlStruct = SqlStruct( + List( + ("myInt", SqlDecimal(Required)), + ("myStruct", SqlStruct(basicFields, Required)) + ) + ) + + val basicOptionalNested: SqlStruct = SqlStruct( + List( + ("myInt", SqlDecimal(Required)), + ("myStruct", SqlStruct(basicFields, Nullable)) + ) + ) + + val basicNestedWithList: SqlStruct = SqlStruct( + List( + ( + "matrix", + SqlStruct( + List( + ("x", SqlDecimal(Required)), + ("y", SqlDecimal(Required)) + ), + Repeated + ) + ) + ) + ) +} diff --git a/jsoncirce/src/test/scala/org/datatools/bigdatatypes/circe/CirceTypeConversionSpec.scala b/jsoncirce/src/test/scala/org/datatools/bigdatatypes/circe/CirceTypeConversionSpec.scala new file mode 100644 index 00000000..995a5c4e --- /dev/null +++ b/jsoncirce/src/test/scala/org/datatools/bigdatatypes/circe/CirceTypeConversionSpec.scala @@ -0,0 +1,44 @@ +package org.datatools.bigdatatypes.circe + +import io.circe.{Json, JsonObject} +import org.datatools.bigdatatypes.basictypes.* +import org.datatools.bigdatatypes.basictypes.SqlType.* +import org.datatools.bigdatatypes.circe.CirceTypeConversion.* +import org.datatools.bigdatatypes.conversions.SqlInstanceConversion +import org.datatools.bigdatatypes.{CirceTestTypes as C, SqlTestTypes as S, UnitSpec} + +/** Reverse conversion, from Circe types to [[SqlType]]s + */ +class CirceTypeConversionSpec extends UnitSpec { + + "Simple Json Type" should "be converted into SqlType" in { + SqlInstanceConversion[Json].getType(Json.fromString("test")) shouldBe SqlString() + } + + "Simple Json Object" should "be converted into SqlStruct" in { + val j = Json.fromJsonObject(JsonObject(("myInt", Json.fromInt(1)))) + val expected = SqlStruct(List(("myInt", SqlDecimal()))) + SqlInstanceConversion[Json].getType(j) shouldBe expected + } + + "Basic Json fields" should "be converted into Basic SqlTypes" in { + SqlInstanceConversion[Json].getType(C.basicTypes) shouldBe S.basicTypes + } + + "Basic Json fields with Arrays" should "be converted into SqlTypes" in { + SqlInstanceConversion[Json].getType(C.basicWithList) shouldBe S.basicWithList + } + + "Basic Json fields with Nested objects" should "be converted into SqlTypes" in { + SqlInstanceConversion[Json].getType(C.basicNested) shouldBe S.basicNested + } + + "Basic Json fields with Nested Arrays" should "be converted into SqlTypes" in { + SqlInstanceConversion[Json].getType(C.basicNestedWithList) shouldBe S.basicNestedWithList + } + + "Extension method asSqlType" should "convert a Json into SqlTypes" in { + C.basicTypes.asSqlType shouldBe S.basicTypes + } + +} diff --git a/spark/src/test/scala/org/datatools/bigdatatypes/SparkTestTypes.scala b/spark/src/test/scala/org/datatools/bigdatatypes/SparkTestTypes.scala index 8f0a58ac..372c227c 100644 --- a/spark/src/test/scala/org/datatools/bigdatatypes/SparkTestTypes.scala +++ b/spark/src/test/scala/org/datatools/bigdatatypes/SparkTestTypes.scala @@ -15,7 +15,7 @@ import org.apache.spark.sql.types.{ TimestampType } -/** Test types from [[TestTypes]] converted to BigQuery. They can be used in multiple tests +/** Test types from [[TestTypes]] converted to Spark. They can be used in multiple tests */ object SparkTestTypes { @@ -30,7 +30,6 @@ object SparkTestTypes { StructField("myString", StringType, nullable = false) ) - /** BigQuery doesn't have a main object, it has a list of fields in the root path */ val basicTypes: StructType = StructType(basicFields) val basicOptionTypes: Seq[StructField] =