Refactor/RenamingClasses

.github/workflows/bigquery-integration.yml

@@ -33,4 +33,5 @@ jobs:
  uses: codecov/codecov-action@v1
  with:
  token: ${{ secrets.CODECOV_TOKEN }}
- verbose: true # optional (default = false)
+ verbose: false # optional (default = false)
+ flags: bq-integration

.github/workflows/ci-tests.yml

@@ -25,4 +25,5 @@ jobs:
  uses: codecov/codecov-action@v1
  with:
  token: ${{ secrets.CODECOV_TOKEN }}
- verbose: true # optional (default = false)
+ verbose: false # optional (default = false)
+ flags: unittests

CONTRIBUTING.md

@@ -9,4 +9,7 @@ Any contribution will be appreciated, regardless if it's a big or small PR, an i
 - Open a PR to `main` branch 
 - Wait for the CI to pass and for a review
 
+## Create a new Type for the library
+There is a complete guide on [how to create a new type](./docs/CreateNewType.md) for the library
+
 For any doubt, don't hesitate to open an [issue](https://github.com/data-tools/big-data-types/issues) or a [discussion](https://github.com/data-tools/big-data-types/discussions)

README.md

@@ -66,7 +66,8 @@ libraryDependencies += "io.github.data-tools" % "big-data-types-core_2.13" % "{v
 Versions for Scala ![Scala 2.12](https://img.shields.io/badge/Scala-2.12-red)
  and ![Scala_2.13](https://img.shields.io/badge/Scala-2.13-red) are available in Maven
 
-
+# How it works
+Check the [complete guide on how to create a new type](./docs/CreateNewType.md) to understand how the library works internally
 
 # BigQuery
 

_config.yml

@@ -0,0 +1 @@
+theme: jekyll-theme-cayman

bigquery/src/main/scala/org/datatools/bigdatatypes/bigquery/BigQueryDefinitions.scala

@@ -32,34 +32,34 @@ private[bigquery] object BigQueryDefinitions {
 
  /** Generates a BigQuery Table Schema given a type A
  */
- def generateSchema[A: BigQueryTypes]: Schema = Schema.of(toJava(BigQueryTypes[A].bigQueryFields))
+ def generateSchema[A: SqlTypeToBigQuery]: Schema = Schema.of(toJava(SqlTypeToBigQuery[A].bigQueryFields))
 
- def generateSchema[A: BigQueryTypes, B: BigQueryTypes]: Schema =
- Schema.of(toJava(BigQueryTypes[A].bigQueryFields ++ BigQueryTypes[B].bigQueryFields))
+ def generateSchema[A: SqlTypeToBigQuery, B: SqlTypeToBigQuery]: Schema =
+ Schema.of(toJava(SqlTypeToBigQuery[A].bigQueryFields ++ SqlTypeToBigQuery[B].bigQueryFields))
 
- def generateSchema[A: BigQueryTypes, B: BigQueryTypes, C: BigQueryTypes]: Schema =
- Schema.of(toJava(BigQueryTypes[A].bigQueryFields ++ BigQueryTypes[B].bigQueryFields ++ BigQueryTypes[C].bigQueryFields))
+ def generateSchema[A: SqlTypeToBigQuery, B: SqlTypeToBigQuery, C: SqlTypeToBigQuery]: Schema =
+ Schema.of(toJava(SqlTypeToBigQuery[A].bigQueryFields ++ SqlTypeToBigQuery[B].bigQueryFields ++ SqlTypeToBigQuery[C].bigQueryFields))
 
- def generateSchema[A: BigQueryTypes, B: BigQueryTypes, C: BigQueryTypes, D: BigQueryTypes]: Schema =
+ def generateSchema[A: SqlTypeToBigQuery, B: SqlTypeToBigQuery, C: SqlTypeToBigQuery, D: SqlTypeToBigQuery]: Schema =
  Schema.of(
- toJava(BigQueryTypes[A].bigQueryFields ++
- BigQueryTypes[B].bigQueryFields ++
- BigQueryTypes[C].bigQueryFields ++
- BigQueryTypes[D].bigQueryFields)
+ toJava(SqlTypeToBigQuery[A].bigQueryFields ++
+ SqlTypeToBigQuery[B].bigQueryFields ++
+ SqlTypeToBigQuery[C].bigQueryFields ++
+ SqlTypeToBigQuery[D].bigQueryFields)
  )
 
- def generateSchema[A: BigQueryTypes, B: BigQueryTypes, C: BigQueryTypes, D: BigQueryTypes, E: BigQueryTypes]: Schema =
+ def generateSchema[A: SqlTypeToBigQuery, B: SqlTypeToBigQuery, C: SqlTypeToBigQuery, D: SqlTypeToBigQuery, E: SqlTypeToBigQuery]: Schema =
  Schema.of(
- toJava(BigQueryTypes[A].bigQueryFields ++
- BigQueryTypes[B].bigQueryFields ++
- BigQueryTypes[C].bigQueryFields ++
- BigQueryTypes[D].bigQueryFields ++
- BigQueryTypes[E].bigQueryFields)
+ toJava(SqlTypeToBigQuery[A].bigQueryFields ++
+ SqlTypeToBigQuery[B].bigQueryFields ++
+ SqlTypeToBigQuery[C].bigQueryFields ++
+ SqlTypeToBigQuery[D].bigQueryFields ++
+ SqlTypeToBigQuery[E].bigQueryFields)
  )
 
  /** For BigQueryTypesInstance
  * Multiples instances not supported for now
  */
- def generateSchema[A: BigQueryTypesInstance](value: A): Schema = Schema.of(toJava(BigQueryTypesInstance[A].bigQueryFields(value)))
+ def generateSchema[A: SqlInstanceToBigQuery](value: A): Schema = Schema.of(toJava(SqlInstanceToBigQuery[A].bigQueryFields(value)))
 
 }

bigquery/src/main/scala/org/datatools/bigdatatypes/bigquery/BigQueryTable.scala

@@ -11,51 +11,51 @@ object BigQueryTable {
 
  /** Create a table without partitions
  */
- def createTable[A: BigQueryTypes](datasetName: String, tableName: String): Either[BigQueryError, Table] = createTable[A](datasetName, tableName, None)
- def createTable[A: BigQueryTypes, B: BigQueryTypes](datasetName: String, tableName: String): Either[BigQueryError, Table] = createTable[A, B](datasetName, tableName, None)
- def createTable[A: BigQueryTypes, B: BigQueryTypes, C: BigQueryTypes](datasetName: String, tableName: String): Either[BigQueryError, Table] = createTable[A, B, C](datasetName, tableName, None)
- def createTable[A: BigQueryTypes, B: BigQueryTypes, C: BigQueryTypes, D: BigQueryTypes](datasetName: String, tableName: String): Either[BigQueryError, Table] = createTable[A, B, C, D](datasetName, tableName, None)
- def createTable[A: BigQueryTypes, B: BigQueryTypes, C: BigQueryTypes, D: BigQueryTypes, E: BigQueryTypes](datasetName: String, tableName: String): Either[BigQueryError, Table] = createTable[A, B, C, D, E](datasetName, tableName, None)
+ def createTable[A: SqlTypeToBigQuery](datasetName: String, tableName: String): Either[BigQueryError, Table] = createTable[A](datasetName, tableName, None)
+ def createTable[A: SqlTypeToBigQuery, B: SqlTypeToBigQuery](datasetName: String, tableName: String): Either[BigQueryError, Table] = createTable[A, B](datasetName, tableName, None)
+ def createTable[A: SqlTypeToBigQuery, B: SqlTypeToBigQuery, C: SqlTypeToBigQuery](datasetName: String, tableName: String): Either[BigQueryError, Table] = createTable[A, B, C](datasetName, tableName, None)
+ def createTable[A: SqlTypeToBigQuery, B: SqlTypeToBigQuery, C: SqlTypeToBigQuery, D: SqlTypeToBigQuery](datasetName: String, tableName: String): Either[BigQueryError, Table] = createTable[A, B, C, D](datasetName, tableName, None)
+ def createTable[A: SqlTypeToBigQuery, B: SqlTypeToBigQuery, C: SqlTypeToBigQuery, D: SqlTypeToBigQuery, E: SqlTypeToBigQuery](datasetName: String, tableName: String): Either[BigQueryError, Table] = createTable[A, B, C, D, E](datasetName, tableName, None)
  /** For Instances as input */
- def createTable[A: BigQueryTypesInstance](value: A, datasetName: String, tableName: String): Either[BigQueryError, Table] = createTable[A](value: A, datasetName, tableName, None)
+ def createTable[A: SqlInstanceToBigQuery](value: A, datasetName: String, tableName: String): Either[BigQueryError, Table] = createTable[A](value: A, datasetName, tableName, None)
 
  /** Create partitioned table
  */
- def createTable[A: BigQueryTypes](datasetName: String, tableName: String, timePartitionColumn: String): Either[BigQueryError, Table] = createTable[A](datasetName, tableName, Some(timePartitionColumn))
- def createTable[A: BigQueryTypes, B: BigQueryTypes](datasetName: String, tableName: String, timePartitionColumn: String): Either[BigQueryError, Table] = createTable[A, B](datasetName, tableName, Some(timePartitionColumn))
- def createTable[A: BigQueryTypes, B: BigQueryTypes, C: BigQueryTypes](datasetName: String, tableName: String, timePartitionColumn: String): Either[BigQueryError, Table] = createTable[A, B, C](datasetName, tableName, Some(timePartitionColumn))
- def createTable[A: BigQueryTypes, B: BigQueryTypes, C: BigQueryTypes, D: BigQueryTypes](datasetName: String, tableName: String, timePartitionColumn: String): Either[BigQueryError, Table] = createTable[A, B, C, D](datasetName, tableName, Some(timePartitionColumn))
- def createTable[A: BigQueryTypes, B: BigQueryTypes, C: BigQueryTypes, D: BigQueryTypes, E: BigQueryTypes](datasetName: String, tableName: String, timePartitionColumn: String): Either[BigQueryError, Table] = createTable[A, B, C, D, E](datasetName, tableName, Some(timePartitionColumn))
+ def createTable[A: SqlTypeToBigQuery](datasetName: String, tableName: String, timePartitionColumn: String): Either[BigQueryError, Table] = createTable[A](datasetName, tableName, Some(timePartitionColumn))
+ def createTable[A: SqlTypeToBigQuery, B: SqlTypeToBigQuery](datasetName: String, tableName: String, timePartitionColumn: String): Either[BigQueryError, Table] = createTable[A, B](datasetName, tableName, Some(timePartitionColumn))
+ def createTable[A: SqlTypeToBigQuery, B: SqlTypeToBigQuery, C: SqlTypeToBigQuery](datasetName: String, tableName: String, timePartitionColumn: String): Either[BigQueryError, Table] = createTable[A, B, C](datasetName, tableName, Some(timePartitionColumn))
+ def createTable[A: SqlTypeToBigQuery, B: SqlTypeToBigQuery, C: SqlTypeToBigQuery, D: SqlTypeToBigQuery](datasetName: String, tableName: String, timePartitionColumn: String): Either[BigQueryError, Table] = createTable[A, B, C, D](datasetName, tableName, Some(timePartitionColumn))
+ def createTable[A: SqlTypeToBigQuery, B: SqlTypeToBigQuery, C: SqlTypeToBigQuery, D: SqlTypeToBigQuery, E: SqlTypeToBigQuery](datasetName: String, tableName: String, timePartitionColumn: String): Either[BigQueryError, Table] = createTable[A, B, C, D, E](datasetName, tableName, Some(timePartitionColumn))
 
  /** For Instances as input */
- def createTable[A: BigQueryTypesInstance](value: A, datasetName: String, tableName: String, timePartitionColumn: String): Either[BigQueryError, Table] = createTable[A](value, datasetName, tableName, Some(timePartitionColumn))
+ def createTable[A: SqlInstanceToBigQuery](value: A, datasetName: String, tableName: String, timePartitionColumn: String): Either[BigQueryError, Table] = createTable[A](value, datasetName, tableName, Some(timePartitionColumn))
 
  /** Create a table in BigQuery
  */
- private def createTable[A: BigQueryTypes](datasetName: String, tableName: String, timePartitionColumn: Option[String]): Either[BigQueryError, Table] = {
+ private def createTable[A: SqlTypeToBigQuery](datasetName: String, tableName: String, timePartitionColumn: Option[String]): Either[BigQueryError, Table] = {
  val schema = generateSchema[A]
  tryTable(TableId.of(datasetName, tableName), generateTableDefinition(schema, timePartitionColumn))
  }
 
- private def createTable[A: BigQueryTypes, B: BigQueryTypes](datasetName: String, tableName: String, timePartitionColumn: Option[String]): Either[BigQueryError, Table] = {
+ private def createTable[A: SqlTypeToBigQuery, B: SqlTypeToBigQuery](datasetName: String, tableName: String, timePartitionColumn: Option[String]): Either[BigQueryError, Table] = {
  val schema = generateSchema[A, B]
  tryTable(TableId.of(datasetName, tableName), generateTableDefinition(schema, timePartitionColumn))
  }
- private def createTable[A: BigQueryTypes, B: BigQueryTypes, C: BigQueryTypes](datasetName: String, tableName: String, timePartitionColumn: Option[String]): Either[BigQueryError, Table] = {
+ private def createTable[A: SqlTypeToBigQuery, B: SqlTypeToBigQuery, C: SqlTypeToBigQuery](datasetName: String, tableName: String, timePartitionColumn: Option[String]): Either[BigQueryError, Table] = {
  val schema = generateSchema[A, B, C]
  tryTable(TableId.of(datasetName, tableName), generateTableDefinition(schema, timePartitionColumn))
  }
- private def createTable[A: BigQueryTypes, B: BigQueryTypes, C: BigQueryTypes, D: BigQueryTypes](datasetName: String, tableName: String, timePartitionColumn: Option[String]): Either[BigQueryError, Table] = {
+ private def createTable[A: SqlTypeToBigQuery, B: SqlTypeToBigQuery, C: SqlTypeToBigQuery, D: SqlTypeToBigQuery](datasetName: String, tableName: String, timePartitionColumn: Option[String]): Either[BigQueryError, Table] = {
  val schema = generateSchema[A, B, C, D]
  tryTable(TableId.of(datasetName, tableName), generateTableDefinition(schema, timePartitionColumn))
  }
- private def createTable[A: BigQueryTypes, B: BigQueryTypes, C: BigQueryTypes, D: BigQueryTypes, E: BigQueryTypes](datasetName: String, tableName: String, timePartitionColumn: Option[String]): Either[BigQueryError, Table] = {
+ private def createTable[A: SqlTypeToBigQuery, B: SqlTypeToBigQuery, C: SqlTypeToBigQuery, D: SqlTypeToBigQuery, E: SqlTypeToBigQuery](datasetName: String, tableName: String, timePartitionColumn: Option[String]): Either[BigQueryError, Table] = {
  val schema = generateSchema[A, B, C, D, E]
  tryTable(TableId.of(datasetName, tableName), generateTableDefinition(schema, timePartitionColumn))
  }
 
  /** For Instances - Only one is accepted for now */
- private def createTable[A: BigQueryTypesInstance](value: A, datasetName: String, tableName: String, timePartitionColumn: Option[String]): Either[BigQueryError, Table] = {
+ private def createTable[A: SqlInstanceToBigQuery](value: A, datasetName: String, tableName: String, timePartitionColumn: Option[String]): Either[BigQueryError, Table] = {
  val schema = generateSchema[A](value)
  tryTable(TableId.of(datasetName, tableName), generateTableDefinition(schema, timePartitionColumn))
  }

bigquery/src/main/scala/org/datatools/bigdatatypes/bigquery/BigQueryTypesInstance.scala → bigquery/src/main/scala/org/datatools/bigdatatypes/bigquery/SqlInstanceToBigQuery.scala

@@ -6,54 +6,54 @@ import org.datatools.bigdatatypes.formats.Formats
 import org.datatools.bigdatatypes.types.basic.SqlType
 
 /** Type class to convert generic SqlTypes received as instance into BigQuery specific fields
- * This uses [[BigQueryTypes]] to create BigQuery Fields
+ * This uses [[SqlTypeToBigQuery]] to create BigQuery Fields
  *
  * @tparam A the type we want to obtain an schema from
  */
-trait BigQueryTypesInstance[A] {
+trait SqlInstanceToBigQuery[A] {
 
  /** @param value an instance of [[A]]
  * @return a list of [[Field]]s that represents [[A]]
  */
  def bigQueryFields(value: A): List[Field]
 }
 
-object BigQueryTypesInstance {
+object SqlInstanceToBigQuery {
 
  /** Summoner method
  */
- def apply[A](implicit a: BigQueryTypesInstance[A]): BigQueryTypesInstance[A] = a
+ def apply[A](implicit a: SqlInstanceToBigQuery[A]): SqlInstanceToBigQuery[A] = a
 
  //TODO change it for the compressed syntax after having everything well documented
  /** Instance derivation via SqlTypeConversion. It uses `getSchema` from BigQueryTypes Type Class
  */
  implicit def fieldsFromSqlInstanceConversion[A: SqlInstanceConversion](implicit
  f: Formats
- ): BigQueryTypesInstance[A] =
- new BigQueryTypesInstance[A] {
+ ): SqlInstanceToBigQuery[A] =
+ new SqlInstanceToBigQuery[A] {
 
  /** @return a list of [[Field]]s that represents [[A]]
  */
  override def bigQueryFields(value: A): List[Field] =
- BigQueryTypes.getSchema(SqlInstanceConversion[A].getType(value))
+ SqlTypeToBigQuery.getSchema(SqlInstanceConversion[A].getType(value))
  }
 
  /** Simplify syntax for SqlType, allows:
- * BigQueryTypesInstance[SqlType].bigQueryFields(mySqlTypeInstance)
+ * SqlInstanceToBigQuery[SqlType].bigQueryFields(mySqlTypeInstance)
  */
- implicit def fieldsFromSqlType(implicit f: Formats): BigQueryTypesInstance[SqlType] =
- new BigQueryTypesInstance[SqlType] {
+ implicit def fieldsFromSqlType(implicit f: Formats): SqlInstanceToBigQuery[SqlType] =
+ new SqlInstanceToBigQuery[SqlType] {
 
  /** @return a list of [[Field]]s that represents [[SqlType]]
  */
  override def bigQueryFields(value: SqlType): List[Field] =
- BigQueryTypes.getSchema(value)
+ SqlTypeToBigQuery.getSchema(value)
  }
 
  /** Allows the syntax myInstance.bigQueryFields for any instance of type A: SqlInstanceConversion
  */
- implicit class InstanceSyntax[A: BigQueryTypesInstance](value: A) {
- def bigQueryFields: List[Field] = BigQueryTypesInstance[A].bigQueryFields(value)
+ implicit class InstanceSyntax[A: SqlInstanceToBigQuery](value: A) {
+ def bigQueryFields: List[Field] = SqlInstanceToBigQuery[A].bigQueryFields(value)
  }
 
  //TODO add another implicit class to resolve mySqlInstance.bigQueryFields

bigquery/src/main/scala/org/datatools/bigdatatypes/bigquery/BigQueryTypes.scala → bigquery/src/main/scala/org/datatools/bigdatatypes/bigquery/SqlTypeToBigQuery.scala

@@ -13,27 +13,27 @@ import org.datatools.bigdatatypes.types.basic._
  *
  * @tparam A the type we want to obtain an schema from
  */
-trait BigQueryTypes[A] {
+trait SqlTypeToBigQuery[A] {
 
  /** @return a list of [[Field]]s that represents [[A]]
  */
  def bigQueryFields: List[Field]
 }
 
-object BigQueryTypes {
+object SqlTypeToBigQuery {
 
  /** Summoner method. Allows the syntax */
- def apply[A](implicit instance: BigQueryTypes[A]): BigQueryTypes[A] = instance
+ def apply[A](implicit instance: SqlTypeToBigQuery[A]): SqlTypeToBigQuery[A] = instance
 
  /** Factory constructor - allows easier construction of instances */
- def instance[A](fs: List[Field]): BigQueryTypes[A] =
- new BigQueryTypes[A] {
+ def instance[A](fs: List[Field]): SqlTypeToBigQuery[A] =
+ new SqlTypeToBigQuery[A] {
  def bigQueryFields: List[Field] = fs
  }
 
  /** Instance derivation via SqlTypeConversion.
  */
- implicit def fieldsFromSqlTypeConversion[A: SqlTypeConversion](implicit f: Formats): BigQueryTypes[A] =
+ implicit def fieldsFromSqlTypeConversion[A: SqlTypeConversion](implicit f: Formats): SqlTypeToBigQuery[A] =
  instance(getSchema(SqlTypeConversion[A].getType))
 
  //TODO improving this and adding all the SqlType options will remove a warning and will allow a syntax like:
@@ -84,6 +84,6 @@ object BigQueryTypes {
  * @tparam A is a Case Class
  */
  implicit class BigQueryFieldSyntax[A <: Product](value: A) {
- def bigQueryFields(implicit a: BigQueryTypes[A]): List[Field] = a.bigQueryFields
+ def bigQueryFields(implicit a: SqlTypeToBigQuery[A]): List[Field] = a.bigQueryFields
  }
 }

bigquery/src/test/scala/org/datatools/bigdatatypes/bigquery/BigQueryTypesInstanceSpec.scala → bigquery/src/test/scala/org/datatools/bigdatatypes/bigquery/SqlTypeToBigQueryInstanceSpec.scala

@@ -2,18 +2,18 @@ package org.datatools.bigdatatypes.bigquery
 
 import com.google.cloud.bigquery.Field
 import org.datatools.bigdatatypes.TestTypes._
-import org.datatools.bigdatatypes.bigquery.BigQueryTypesInstance.InstanceSyntax
+import org.datatools.bigdatatypes.bigquery.SqlInstanceToBigQuery.InstanceSyntax
 import org.datatools.bigdatatypes.conversions.SqlTypeConversion
 import org.datatools.bigdatatypes.conversions.SqlTypeConversion._
 import org.datatools.bigdatatypes.formats.Formats.implicitDefaultFormats
 import org.datatools.bigdatatypes.types.basic.SqlType
 import org.datatools.bigdatatypes.{BigQueryTestTypes, UnitSpec}
 
 /** These tests defines how to convert an SqlType instance into BigQueryFields
- * Not all the possible `TestTypes` are tested as they use the same code tested in [[BigQueryTypesSpec]]
+ * Not all the possible `TestTypes` are tested as they use the same code tested in [[SqlTypeToBigQuerySpec]]
  * Cross tests examples in Example module should define how to transform other types into BigQueryFields
  */
-class BigQueryTypesInstanceSpec extends UnitSpec {
+class SqlTypeToBigQueryInstanceSpec extends UnitSpec {
 
  behavior of "BigQueryTypesInstanceSpec"
 
@@ -24,19 +24,19 @@ class BigQueryTypesInstanceSpec extends UnitSpec {
 
  "basic case class" should "be converted into BigQueryFields" in {
  val sql: SqlType = SqlTypeConversion[BasicTypes].getType
- val fields: Seq[Field] = BigQueryTypesInstance[SqlType].bigQueryFields(sql)
+ val fields: Seq[Field] = SqlInstanceToBigQuery[SqlType].bigQueryFields(sql)
  fields shouldBe BigQueryTestTypes.basicTypes
  }
 
  "Case Class with Option" should "be converted into BigQueryFields with nullable" in {
  val sql: SqlType = SqlTypeConversion[BasicOption].getType
- val fields: Seq[Field] = BigQueryTypesInstance[SqlType].bigQueryFields(sql)
+ val fields: Seq[Field] = SqlInstanceToBigQuery[SqlType].bigQueryFields(sql)
  fields shouldBe BigQueryTestTypes.basicOption
  }
 
  "Case class with Struct List" should "be converted into Repeated BigQueryFields" in {
  val sql: SqlType = SqlTypeConversion[ListOfStruct].getType
- val fields: Seq[Field] = BigQueryTypesInstance[SqlType].bigQueryFields(sql)
+ val fields: Seq[Field] = SqlInstanceToBigQuery[SqlType].bigQueryFields(sql)
  fields shouldBe BigQueryTestTypes.basicNestedWithList
  }
 }