forked from data-tools/big-data-types
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* improving docs + Circe module in build + small comment improvements * Some conversions from Circe to SqlType + some test types * Circe Conversion working for arrays * Circe docs + examples
- Loading branch information
1 parent
3ff1664
commit 7b271fd
Showing
16 changed files
with
546 additions
and
36 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1,5 @@ | ||
| //used to build Sonatype releases | ||
| lazy val versionNumber = "1.1.2" | ||
| lazy val versionNumber = "1.2.0" | ||
| lazy val projectName = "big-data-types" | ||
| version := versionNumber | ||
| name := projectName | ||
|
|
@@ -26,7 +26,7 @@ lazy val publishSettings = Seq( | |
| ScmInfo(url("https://github.com/data-tools/big-data-types"), "[email protected]:data-tools/big-data-types.git") | ||
| ), | ||
| developers := List(Developer("JavierMonton", "Javier Monton", "", url("https://github.com/JavierMonton"))), | ||
| licenses := Seq("APL2" -> url("http:https://www.apache.org/licenses/LICENSE-2.0.txt")), | ||
| licenses := Seq("APL2" -> url("https:https://www.apache.org/licenses/LICENSE-2.0.txt")), | ||
| publishMavenStyle := true | ||
| ) | ||
|
|
||
|
|
@@ -69,6 +69,13 @@ lazy val cassandraDependencies = Seq( | |
| scalatest % Test | ||
| ) | ||
|
|
||
| val circeVersion = "0.14.1" | ||
| lazy val jsonCirceDependencies = Seq( | ||
| "io.circe" %% "circe-core", | ||
| "io.circe" %% "circe-generic", | ||
| "io.circe" %% "circe-parser" | ||
| ).map(_ % circeVersion) | ||
|
|
||
| lazy val scalatest = "org.scalatest" %% "scalatest" % "3.2.11" | ||
|
|
||
| //Project settings | ||
|
|
@@ -80,6 +87,7 @@ lazy val root = (project in file(".")) | |
| bigquery, | ||
| spark, | ||
| cassandra, | ||
| jsonCirce, | ||
| examples | ||
| ) | ||
|
|
||
|
|
@@ -135,6 +143,18 @@ lazy val cassandra = (project in file("cassandra")) | |
| ) | ||
| .dependsOn(core % "test->test;compile->compile") | ||
|
|
||
| lazy val jsonCirce = (project in file("jsoncirce")) | ||
| .configs(IntegrationTest) | ||
| .settings( | ||
| name := projectName + "-circe", | ||
| publishSettings, | ||
| scalacOptions ++= scalacCommon, | ||
| crossScalaVersions := supportedScalaVersions, | ||
| crossVersionSharedSources, | ||
| libraryDependencies ++= jsonCirceDependencies | ||
| ) | ||
| .dependsOn(core % "test->test;compile->compile") | ||
|
|
||
| // Examples module for testing, with all modules included, not built | ||
| lazy val examples = (project in file("examples")) | ||
| .settings( | ||
|
|
@@ -147,6 +167,7 @@ lazy val examples = (project in file("examples")) | |
| .dependsOn(core % "test->test;compile->compile") | ||
| .dependsOn(bigquery % "test->test;compile->compile") | ||
| .dependsOn(cassandra % "test->test;compile->compile") | ||
| .dependsOn(jsonCirce % "test->test;compile->compile") | ||
| .settings( | ||
| noPublishSettings, | ||
| crossScalaVersions := List(scala212, scala213), | ||
|
|
||
6 changes: 3 additions & 3 deletions
6
core/src/test/scala_2/org/datatools/bigdatatypes/TestTypes.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,68 @@ | ||
| --- | ||
| sidebar_position: 6 | ||
| --- | ||
| # Circe (JSON) | ||
|
|
||
| [Circe](https://circe.github.io/circe/) is a JSON library for Scala. | ||
|
|
||
| The Circe module of this library allows to convert `Json` objects (from Circe) to any other type in the library. | ||
| :::caution | ||
| For now only conversions from Circe to other types are available. Other types to Circe are not ready yet. | ||
| ::: | ||
|
|
||
| :::info | ||
| Json objects do not have very concrete types, meaning that `number` is a type, | ||
| but more specific types like `integer`, `float` or others do not exists. | ||
| Because of that, any conversion between types will convert `number` into `Decimal` types, | ||
| as `Decimal` is the only one that can ensure the precision of any arbitrary number | ||
| ::: | ||
| <details><summary>About Circe and private types</summary></details> | ||
| <p> | ||
| Circe has more specific types than `JNumber`, like `JLong`, `JDouble` and other, | ||
| but all of them are private to Circe itself, so we can not use them, not even for matching types during conversions. | ||
| In any case, even if we were able to use them, when parsing a JSON string (probably most of the cases) | ||
| we can not detect the specific types | ||
| </p> | ||
|
|
||
| ```scala | ||
| import io.Circe.Json | ||
| import org.datatools.bigdatatypes.circe.CirceTypeConversion.* | ||
| import org.datatools.bigdatatypes.formats.Formats.implicitDefaultFormats | ||
| import org.datatools.bigdatatypes.spark.SqlInstanceToSpark.InstanceSyntax | ||
| import org.datatools.bigdatatypes.bigquery.SqlInstanceToBigQuery.* | ||
|
|
||
| val circeJson: Json = Json.fromFields(List( | ||
| ("id", Json.fromString("test")), | ||
| ("foo", Json.fromString("test")), | ||
| ("bar", Json.fromInt(1)) | ||
| )) | ||
|
|
||
| val sparkSchema: StructType = circeJson.asSparkSchema | ||
| val bqSchema: Schema = circeJson.asBigQuery.schema | ||
| ``` | ||
|
|
||
| Or if you do it from a JSON parsed using Circe: | ||
| ```scala | ||
| import io.circe._, io.circe.parser._ | ||
|
|
||
| import org.datatools.bigdatatypes.circe.CirceTypeConversion.* | ||
| import org.datatools.bigdatatypes.formats.Formats.implicitDefaultFormats | ||
| import org.datatools.bigdatatypes.spark.SqlInstanceToSpark.InstanceSyntax | ||
| import org.datatools.bigdatatypes.bigquery.SqlInstanceToBigQuery.* | ||
|
|
||
| val rawJson: String = """ | ||
| { | ||
| "foo": "bar", | ||
| "baz": 123, | ||
| "list of stuff": [ 4, 5, 6 ] | ||
| } | ||
| """ | ||
| val parseResult = parse(rawJson) | ||
| // parseResult: Either[ParsingFailure, Json] | ||
| val sparkSchema = parseResult.map(j => j.asSparkSchema) | ||
| // sparkSchema: Either[ParsingFailure, StructType] | ||
| val bqSchema = parseResult.map(j => j.asBigQuery.schema) | ||
| // bqSchema: Either[ParsingFailure, Schema] | ||
| ``` | ||
|
|
||
| --- |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.