Cover remaining coverage gaps (formats on Scala 2, instance derivations, table overloads)

bigquery/src/test/scala/org/datatools/bigdatatypes/bigquery/BigQueryDefinitionsSpec.scala

@@ -0,0 +1,33 @@
+package org.datatools.bigdatatypes.bigquery
+
+import com.google.cloud.bigquery.{Schema, TimePartitioning}
+import org.datatools.bigdatatypes.TestTypes.BasicTypes
+import org.datatools.bigdatatypes.UnitSpec
+import org.datatools.bigdatatypes.bigquery.BigQueryDefinitions.{
+  generateTableDefinition,
+  generateTimePartitionColumn
+}
+import org.datatools.bigdatatypes.bigquery.JavaConverters.toJava
+import org.datatools.bigdatatypes.formats.Formats.implicitDefaultFormats
+
+class BigQueryDefinitionsSpec extends UnitSpec {
+
+  private val schema: Schema = Schema.of(toJava(SqlTypeToBigQuery[BasicTypes].bigQueryFields))
+
+  "generateTimePartitionColumn" should "create a DAY partition on the column" in {
+    val partition = generateTimePartitionColumn("myDate")
+    partition.getField shouldBe "myDate"
+    partition.getType shouldBe TimePartitioning.Type.DAY
+  }
+
+  "generateTableDefinition" should "keep the schema without partition" in {
+    generateTableDefinition(schema, None).getSchema shouldBe schema
+  }
+
+  it should "add the partition column when defined" in {
+    val definition = generateTableDefinition(schema, Some("myDate"))
+    definition.getSchema shouldBe schema
+    definition.getTimePartitioning.getField shouldBe "myDate"
+  }
+
+}

bigquery/src/test/scala/org/datatools/bigdatatypes/bigquery/BigQueryTableUnitSpec.scala

@@ -1,9 +1,13 @@
 package org.datatools.bigdatatypes.bigquery
 
-import org.datatools.bigdatatypes.TestTypes.BasicTypes
+import com.google.cloud.bigquery.Field.Mode
+import com.google.cloud.bigquery.{Field, Schema}
+import org.datatools.bigdatatypes.TestTypes.{BasicList, BasicOption, BasicStruct, BasicTypes, ExtendedTypes}
 import org.datatools.bigdatatypes.UnitSpec
+import org.datatools.bigdatatypes.bigquery.BigQueryTypeConversion.{field, schema}
 import org.datatools.bigdatatypes.conversions.SqlTypeConversion
 import org.datatools.bigdatatypes.formats.Formats.implicitDefaultFormats
+import com.google.cloud.bigquery.StandardSQLTypeName
 
 class BigQueryTableUnitSpec extends UnitSpec {
 
@@ -12,4 +16,30 @@ class BigQueryTableUnitSpec extends UnitSpec {
     BigQueryTable.createTable(sql, "test", "sqlType_table").isLeft shouldBe true
   }
 
+  "Typed createTable" should "fail without Service Account for every arity" in {
+    BigQueryTable.createTable[BasicTypes]("test", "t").isLeft shouldBe true
+    BigQueryTable.createTable[BasicTypes, BasicStruct]("test", "t").isLeft shouldBe true
+    BigQueryTable.createTable[BasicTypes, BasicStruct, BasicList]("test", "t").isLeft shouldBe true
+    BigQueryTable.createTable[BasicTypes, BasicStruct, BasicList, BasicOption]("test", "t").isLeft shouldBe true
+    BigQueryTable
+      .createTable[BasicTypes, BasicStruct, BasicList, BasicOption, ExtendedTypes]("test", "t")
+      .isLeft shouldBe true
+  }
+
+  "Partitioned createTable" should "fail without Service Account for every arity" in {    BigQueryTable.createTable[BasicTypes]("test", "t", "part").isLeft shouldBe true
+    BigQueryTable.createTable[BasicTypes, BasicStruct]("test", "t", "part").isLeft shouldBe true
+    BigQueryTable.createTable[BasicTypes, BasicStruct, BasicList]("test", "t", "part").isLeft shouldBe true
+    BigQueryTable.createTable[BasicTypes, BasicStruct, BasicList, BasicOption]("test", "t", "part").isLeft shouldBe true
+    BigQueryTable
+      .createTable[BasicTypes, BasicStruct, BasicList, BasicOption, ExtendedTypes]("test", "t", "part")
+      .isLeft shouldBe true
+  }
+
+  "Instance partitioned createTable" should "fail without Service Account" in {
+    val myField =
+      Field.newBuilder("myInt", StandardSQLTypeName.INT64).setMode(Mode.REQUIRED).build()
+    val bqSchema = Schema.of(myField)
+    BigQueryTable.createTable(bqSchema, "test", "t", "part").isLeft shouldBe true
+  }
+
 }

bigquery/src/test/scala/org/datatools/bigdatatypes/bigquery/BigQueryTypeConversionSpec.scala

@@ -16,7 +16,8 @@ import org.datatools.bigdatatypes.UnitSpec
 import org.datatools.bigdatatypes.basictypes.SqlType
 import org.datatools.bigdatatypes.basictypes.SqlType.*
 import org.datatools.bigdatatypes.basictypes.SqlTypeMode.Required
-import org.datatools.bigdatatypes.bigquery.BigQueryTypeConversion.{field, intType, FieldTypeSyntax, SchemaFieldSyntax}
+import org.datatools.bigdatatypes.bigquery.BigQueryTypeConversion.{field, intType, schema, FieldTypeSyntax, SchemaFieldSyntax}
+import org.datatools.bigdatatypes.bigquery.SqlInstanceToBigQuery.*
 import org.datatools.bigdatatypes.bigquery.JavaConverters.toJava
 import org.datatools.bigdatatypes.conversions.{SqlInstanceConversion, SqlTypeConversion}
 import org.datatools.bigdatatypes.formats.{DefaultFormats, Formats}
@@ -98,4 +99,17 @@ class BigQueryTypeConversionSpec extends UnitSpec {
     bqSchema.asSqlType shouldBe reduceBQTypes(SqlTypeConversion[ExtendedTypes].getType)
   }
 
+  "Schema instance" should "be converted into BigQuery fields" in {
+    val myField = Field.newBuilder("myInt", StandardSQLTypeName.INT64).setMode(Mode.REQUIRED).build()
+    val bqSchema = Schema.of(myField)
+    SqlInstanceToBigQuery[Schema].bigQueryFields(bqSchema) shouldBe SqlTypeToBigQuery.getSchema(
+      SqlInstanceConversion[Schema].getType(bqSchema)
+    )
+  }
+
+  "Field list" should "be converted into a Schema using extension method" in {
+    val myField = Field.newBuilder("myInt", StandardSQLTypeName.INT64).setMode(Mode.REQUIRED).build()
+    List(myField).schema shouldBe Schema.of(myField)
+  }
+
 }

cassandra/src/test/scala/org/datatools/bigdatatypes/cassandra/CassandraTablesSpec.scala

@@ -1,9 +1,12 @@
 package org.datatools.bigdatatypes.cassandra
 
 import com.datastax.oss.driver.api.querybuilder.schema.CreateTable
+import com.datastax.oss.driver.api.core.`type`.{DataType, DataTypes}
 import org.datatools.bigdatatypes.TestTypes.BasicTypes
 import org.datatools.bigdatatypes.UnitSpec
 import org.datatools.bigdatatypes.basictypes.SqlType
+import org.datatools.bigdatatypes.basictypes.SqlType._
+import org.datatools.bigdatatypes.basictypes.SqlTypeMode.Required
 import org.datatools.bigdatatypes.cassandra.CassandraTables.AsCassandraProductSyntax
 import org.datatools.bigdatatypes.conversions.SqlTypeConversion
 import org.datatools.bigdatatypes.formats.Formats.implicitDefaultFormats
@@ -26,4 +29,43 @@ class CassandraTablesSpec extends UnitSpec {
     val table = instance.asCassandra("TestTable", "myLong")
     table.toString shouldBe "CREATE TABLE testtable (myint int,mylong bigint PRIMARY KEY,myfloat float,mydouble double,mydecimal decimal,myboolean boolean,mystring text)"
   }
+
+  it should "create the primary key as Text when it is missing" in {
+    val table: CreateTable = CassandraTables.table[BasicTypes]("TestTable", "unknownKey")
+    table.toString should include("unknownkey text")
+    table.toString should include("PRIMARY KEY")
+  }
+
+  it should "be created from tuple schema" in {
+    import CassandraTypeConversion.cassandraTupleType
+    val table: CreateTable =
+      CassandraTables.table[(String, DataType)](("myLong", DataTypes.BIGINT), "TestTable", "myLong")
+    table.toString shouldBe "CREATE TABLE testtable (mylong bigint PRIMARY KEY)"
+  }
+
+  it should "be created from instance using extension method" in {
+    import CassandraTables.AsCassandraInstanceSyntax
+    import CassandraTypeConversion.cassandraCreateTable
+    val source: CreateTable = CassandraTables.table[BasicTypes]("TestTable", "myLong")
+    val table = source.asCassandra("TestTable2", "myLong")
+    table.toString should include("testtable2")
+    table.toString should include("mylong bigint")
+  }
+
+  it should "convert a CreateTable back into SqlType" in {
+    val table: CreateTable = CassandraTables.table[BasicTypes]("TestTable", "myLong")
+    // CQL identifiers are lowercased by the driver
+    val expected = SqlStruct(
+      List(
+        ("myint", SqlInt(Required)),
+        ("mylong", SqlLong(Required)),
+        ("myfloat", SqlFloat(Required)),
+        ("mydouble", SqlDouble(Required)),
+        ("mydecimal", SqlDecimal(Required)),
+        ("myboolean", SqlBool(Required)),
+        ("mystring", SqlString(Required))
+      )
+    )
+    CassandraTypeConversion.cassandraCreateTable.getType(table) shouldBe expected
+  }
 }

cassandra/src/test/scala/org/datatools/bigdatatypes/cassandra/SqlInstanceToCassandraSpec.scala

@@ -2,11 +2,14 @@ package org.datatools.bigdatatypes.cassandra
 
 import org.datatools.bigdatatypes.TestTypes.BasicTypes
 import org.datatools.bigdatatypes.basictypes.SqlType
+import org.datatools.bigdatatypes.cassandra.CassandraTypeConversion.cassandraTupleType
 import org.datatools.bigdatatypes.formats.{DefaultFormats, Formats}
 import org.datatools.bigdatatypes.{CassandraTestTypes, UnitSpec}
 import org.datatools.bigdatatypes.cassandra.SqlInstanceToCassandra.*
 import org.datatools.bigdatatypes.conversions.SqlTypeConversion
 
+import com.datastax.oss.driver.api.core.`type`.{DataType, DataTypes}
+
 class SqlInstanceToCassandraSpec extends UnitSpec {
 
   implicit val defaultFormats: Formats = DefaultFormats
@@ -27,4 +30,9 @@ class SqlInstanceToCassandraSpec extends UnitSpec {
     sql.asCassandra shouldBe CassandraTestTypes.basicFields
   }
 
+  "Tuple instance" should "be converted into Cassandra tuples" in {
+    val tuple: (String, DataType) = ("myLong", DataTypes.BIGINT)
+    SqlInstanceToCassandra[(String, DataType)].cassandraFields(tuple) shouldBe List(tuple)
+  }
+
 }

core/src/test/scala_2/org/datatools/bigdatatypes/formats/FormatsSpec.scala

@@ -1,9 +1,47 @@
 package org.datatools.bigdatatypes.formats
 
 import org.datatools.bigdatatypes.UnitSpec
+import org.datatools.bigdatatypes.basictypes.SqlType._
+import org.datatools.bigdatatypes.basictypes.SqlTypeMode._
 
 class FormatsSpec extends UnitSpec {
 
-  "Default Formats" should "do nothing to keys" in {}
+  behavior of "FormatsSpec"
+
+  "Default Formats" should "leave keys unchanged" in {
+    DefaultFormats.transformKey("myValue", SqlString()) shouldBe "myValue"
+    DefaultFormats.transformKey("myValue", SqlInt()) shouldBe "myValue"
+  }
+
+  "Default Formats" should "use BigDecimal precision 10 scale 0" in {
+    DefaultFormats.bigDecimal shouldBe DefaultFormats.BigDecimalPrecision(10, 0)
+  }
+
+  "Base Formats" should "leave keys unchanged by default" in {
+    val formats = new Formats {}
+    formats.transformKey("myValue", SqlString()) shouldBe "myValue"
+  }
+
+  "Snakify Formats" should "convert camelCase to snake_case" in {
+    SnakifyFormats.transformKey("myValue", SqlString()) shouldBe "my_value"
+  }
+
+  "Snakify Formats" should "handle acronyms" in {
+    SnakifyFormats.transformKey("myURLValue", SqlString()) shouldBe "my_url_value"
+  }
+
+  "KeyTypeExample Formats" should "prefix booleans with is_" in {
+    KeyTypeExampleFormats.transformKey("active", SqlBool()) shouldBe "is_active"
+  }
+
+  "KeyTypeExample Formats" should "suffix dates and timestamps with _at" in {
+    KeyTypeExampleFormats.transformKey("birth", SqlDate()) shouldBe "birth_at"
+    KeyTypeExampleFormats.transformKey("created", SqlTimestamp()) shouldBe "created_at"
+  }
+
+  "KeyTypeExample Formats" should "leave other types unchanged" in {
+    KeyTypeExampleFormats.transformKey("name", SqlString()) shouldBe "name"
+    KeyTypeExampleFormats.transformKey("count", SqlInt()) shouldBe "count"
+  }
 
 }

spark/src/test/scala/org/datatools/bigdatatypes/spark/SparkTypeConversionSpec.scala

@@ -9,6 +9,7 @@ import org.datatools.bigdatatypes.basictypes.SqlTypeMode.*
 import org.datatools.bigdatatypes.conversions.{SqlInstanceConversion, SqlTypeConversion}
 import org.datatools.bigdatatypes.formats.Formats.implicitDefaultFormats
 import org.datatools.bigdatatypes.spark.SparkTypeConversion.*
+import org.datatools.bigdatatypes.spark.SqlInstanceToSpark.*
 
 /** Reverse conversion, from Spark types to [[SqlType]]s
   */
@@ -81,4 +82,29 @@ class SparkTypeConversionSpec extends UnitSpec {
     sqlType shouldBe extendedTypes
   }
 
+  "StructType instance" should "be converted into Spark fields" in {
+    val sf = StructField("myInt", IntegerType, nullable = false)
+    val sf2 = StructField("myString", StringType, nullable = true)
+    val st = StructType(List(sf, sf2))
+    SqlInstanceToSpark[StructType].sparkFields(st) shouldBe SqlTypeToSpark.getSchema(
+      SqlInstanceConversion[StructType].getType(st)
+    )
+  }
+
+  "StructType instance" should "be converted using extension methods" in {
+    val sf = StructField("myInt", IntegerType, nullable = false)
+    val sf2 = StructField("myString", StringType, nullable = true)
+    val st = StructType(List(sf, sf2))
+    st.asSparkFields shouldBe SqlInstanceToSpark[StructType].sparkFields(st)
+    st.asSparkSchema shouldBe StructType(SqlInstanceToSpark[StructType].sparkFields(st))
+  }
+
+  "SparkSchemas" should "build fields and schemas from instances" in {
+    val sf = StructField("myInt", IntegerType, nullable = false)
+    val sf2 = StructField("myString", StringType, nullable = true)
+    val st = StructType(List(sf, sf2))
+    SparkSchemas.fields(st) shouldBe SqlInstanceToSpark[StructType].sparkFields(st)
+    SparkSchemas.schema(st) shouldBe StructType(SqlInstanceToSpark[StructType].sparkFields(st))
+  }
+
 }