diff --git a/bigquery/src/test/scala/org/datatools/bigdatatypes/bigquery/BigQueryDefinitionsSpec.scala b/bigquery/src/test/scala/org/datatools/bigdatatypes/bigquery/BigQueryDefinitionsSpec.scala new file mode 100644 index 00000000..1fec8731 --- /dev/null +++ b/bigquery/src/test/scala/org/datatools/bigdatatypes/bigquery/BigQueryDefinitionsSpec.scala @@ -0,0 +1,33 @@ +package org.datatools.bigdatatypes.bigquery + +import com.google.cloud.bigquery.{Schema, TimePartitioning} +import org.datatools.bigdatatypes.TestTypes.BasicTypes +import org.datatools.bigdatatypes.UnitSpec +import org.datatools.bigdatatypes.bigquery.BigQueryDefinitions.{ + generateTableDefinition, + generateTimePartitionColumn +} +import org.datatools.bigdatatypes.bigquery.JavaConverters.toJava +import org.datatools.bigdatatypes.formats.Formats.implicitDefaultFormats + +class BigQueryDefinitionsSpec extends UnitSpec { + + private val schema: Schema = Schema.of(toJava(SqlTypeToBigQuery[BasicTypes].bigQueryFields)) + + "generateTimePartitionColumn" should "create a DAY partition on the column" in { + val partition = generateTimePartitionColumn("myDate") + partition.getField shouldBe "myDate" + partition.getType shouldBe TimePartitioning.Type.DAY + } + + "generateTableDefinition" should "keep the schema without partition" in { + generateTableDefinition(schema, None).getSchema shouldBe schema + } + + it should "add the partition column when defined" in { + val definition = generateTableDefinition(schema, Some("myDate")) + definition.getSchema shouldBe schema + definition.getTimePartitioning.getField shouldBe "myDate" + } + +} diff --git a/bigquery/src/test/scala/org/datatools/bigdatatypes/bigquery/BigQueryTableUnitSpec.scala b/bigquery/src/test/scala/org/datatools/bigdatatypes/bigquery/BigQueryTableUnitSpec.scala index f43ea00d..b78b762a 100644 --- a/bigquery/src/test/scala/org/datatools/bigdatatypes/bigquery/BigQueryTableUnitSpec.scala +++ b/bigquery/src/test/scala/org/datatools/bigdatatypes/bigquery/BigQueryTableUnitSpec.scala @@ -1,9 +1,13 @@ package org.datatools.bigdatatypes.bigquery -import org.datatools.bigdatatypes.TestTypes.BasicTypes +import com.google.cloud.bigquery.Field.Mode +import com.google.cloud.bigquery.{Field, Schema} +import org.datatools.bigdatatypes.TestTypes.{BasicList, BasicOption, BasicStruct, BasicTypes, ExtendedTypes} import org.datatools.bigdatatypes.UnitSpec +import org.datatools.bigdatatypes.bigquery.BigQueryTypeConversion.{field, schema} import org.datatools.bigdatatypes.conversions.SqlTypeConversion import org.datatools.bigdatatypes.formats.Formats.implicitDefaultFormats +import com.google.cloud.bigquery.StandardSQLTypeName class BigQueryTableUnitSpec extends UnitSpec { @@ -12,4 +16,30 @@ class BigQueryTableUnitSpec extends UnitSpec { BigQueryTable.createTable(sql, "test", "sqlType_table").isLeft shouldBe true } + "Typed createTable" should "fail without Service Account for every arity" in { + BigQueryTable.createTable[BasicTypes]("test", "t").isLeft shouldBe true + BigQueryTable.createTable[BasicTypes, BasicStruct]("test", "t").isLeft shouldBe true + BigQueryTable.createTable[BasicTypes, BasicStruct, BasicList]("test", "t").isLeft shouldBe true + BigQueryTable.createTable[BasicTypes, BasicStruct, BasicList, BasicOption]("test", "t").isLeft shouldBe true + BigQueryTable + .createTable[BasicTypes, BasicStruct, BasicList, BasicOption, ExtendedTypes]("test", "t") + .isLeft shouldBe true + } + + "Partitioned createTable" should "fail without Service Account for every arity" in { BigQueryTable.createTable[BasicTypes]("test", "t", "part").isLeft shouldBe true + BigQueryTable.createTable[BasicTypes, BasicStruct]("test", "t", "part").isLeft shouldBe true + BigQueryTable.createTable[BasicTypes, BasicStruct, BasicList]("test", "t", "part").isLeft shouldBe true + BigQueryTable.createTable[BasicTypes, BasicStruct, BasicList, BasicOption]("test", "t", "part").isLeft shouldBe true + BigQueryTable + .createTable[BasicTypes, BasicStruct, BasicList, BasicOption, ExtendedTypes]("test", "t", "part") + .isLeft shouldBe true + } + + "Instance partitioned createTable" should "fail without Service Account" in { + val myField = + Field.newBuilder("myInt", StandardSQLTypeName.INT64).setMode(Mode.REQUIRED).build() + val bqSchema = Schema.of(myField) + BigQueryTable.createTable(bqSchema, "test", "t", "part").isLeft shouldBe true + } + } diff --git a/bigquery/src/test/scala/org/datatools/bigdatatypes/bigquery/BigQueryTypeConversionSpec.scala b/bigquery/src/test/scala/org/datatools/bigdatatypes/bigquery/BigQueryTypeConversionSpec.scala index 9c47753c..88b558bb 100644 --- a/bigquery/src/test/scala/org/datatools/bigdatatypes/bigquery/BigQueryTypeConversionSpec.scala +++ b/bigquery/src/test/scala/org/datatools/bigdatatypes/bigquery/BigQueryTypeConversionSpec.scala @@ -16,7 +16,8 @@ import org.datatools.bigdatatypes.UnitSpec import org.datatools.bigdatatypes.basictypes.SqlType import org.datatools.bigdatatypes.basictypes.SqlType.* import org.datatools.bigdatatypes.basictypes.SqlTypeMode.Required -import org.datatools.bigdatatypes.bigquery.BigQueryTypeConversion.{field, intType, FieldTypeSyntax, SchemaFieldSyntax} +import org.datatools.bigdatatypes.bigquery.BigQueryTypeConversion.{field, intType, schema, FieldTypeSyntax, SchemaFieldSyntax} +import org.datatools.bigdatatypes.bigquery.SqlInstanceToBigQuery.* import org.datatools.bigdatatypes.bigquery.JavaConverters.toJava import org.datatools.bigdatatypes.conversions.{SqlInstanceConversion, SqlTypeConversion} import org.datatools.bigdatatypes.formats.{DefaultFormats, Formats} @@ -98,4 +99,17 @@ class BigQueryTypeConversionSpec extends UnitSpec { bqSchema.asSqlType shouldBe reduceBQTypes(SqlTypeConversion[ExtendedTypes].getType) } + "Schema instance" should "be converted into BigQuery fields" in { + val myField = Field.newBuilder("myInt", StandardSQLTypeName.INT64).setMode(Mode.REQUIRED).build() + val bqSchema = Schema.of(myField) + SqlInstanceToBigQuery[Schema].bigQueryFields(bqSchema) shouldBe SqlTypeToBigQuery.getSchema( + SqlInstanceConversion[Schema].getType(bqSchema) + ) + } + + "Field list" should "be converted into a Schema using extension method" in { + val myField = Field.newBuilder("myInt", StandardSQLTypeName.INT64).setMode(Mode.REQUIRED).build() + List(myField).schema shouldBe Schema.of(myField) + } + } diff --git a/cassandra/src/test/scala/org/datatools/bigdatatypes/cassandra/CassandraTablesSpec.scala b/cassandra/src/test/scala/org/datatools/bigdatatypes/cassandra/CassandraTablesSpec.scala index b4376821..c4dd1dc5 100644 --- a/cassandra/src/test/scala/org/datatools/bigdatatypes/cassandra/CassandraTablesSpec.scala +++ b/cassandra/src/test/scala/org/datatools/bigdatatypes/cassandra/CassandraTablesSpec.scala @@ -1,9 +1,12 @@ package org.datatools.bigdatatypes.cassandra import com.datastax.oss.driver.api.querybuilder.schema.CreateTable +import com.datastax.oss.driver.api.core.`type`.{DataType, DataTypes} import org.datatools.bigdatatypes.TestTypes.BasicTypes import org.datatools.bigdatatypes.UnitSpec import org.datatools.bigdatatypes.basictypes.SqlType +import org.datatools.bigdatatypes.basictypes.SqlType._ +import org.datatools.bigdatatypes.basictypes.SqlTypeMode.Required import org.datatools.bigdatatypes.cassandra.CassandraTables.AsCassandraProductSyntax import org.datatools.bigdatatypes.conversions.SqlTypeConversion import org.datatools.bigdatatypes.formats.Formats.implicitDefaultFormats @@ -26,4 +29,43 @@ class CassandraTablesSpec extends UnitSpec { val table = instance.asCassandra("TestTable", "myLong") table.toString shouldBe "CREATE TABLE testtable (myint int,mylong bigint PRIMARY KEY,myfloat float,mydouble double,mydecimal decimal,myboolean boolean,mystring text)" } + + it should "create the primary key as Text when it is missing" in { + val table: CreateTable = CassandraTables.table[BasicTypes]("TestTable", "unknownKey") + table.toString should include("unknownkey text") + table.toString should include("PRIMARY KEY") + } + + it should "be created from tuple schema" in { + import CassandraTypeConversion.cassandraTupleType + val table: CreateTable = + CassandraTables.table[(String, DataType)](("myLong", DataTypes.BIGINT), "TestTable", "myLong") + table.toString shouldBe "CREATE TABLE testtable (mylong bigint PRIMARY KEY)" + } + + it should "be created from instance using extension method" in { + import CassandraTables.AsCassandraInstanceSyntax + import CassandraTypeConversion.cassandraCreateTable + val source: CreateTable = CassandraTables.table[BasicTypes]("TestTable", "myLong") + val table = source.asCassandra("TestTable2", "myLong") + table.toString should include("testtable2") + table.toString should include("mylong bigint") + } + + it should "convert a CreateTable back into SqlType" in { + val table: CreateTable = CassandraTables.table[BasicTypes]("TestTable", "myLong") + // CQL identifiers are lowercased by the driver + val expected = SqlStruct( + List( + ("myint", SqlInt(Required)), + ("mylong", SqlLong(Required)), + ("myfloat", SqlFloat(Required)), + ("mydouble", SqlDouble(Required)), + ("mydecimal", SqlDecimal(Required)), + ("myboolean", SqlBool(Required)), + ("mystring", SqlString(Required)) + ) + ) + CassandraTypeConversion.cassandraCreateTable.getType(table) shouldBe expected + } } diff --git a/cassandra/src/test/scala/org/datatools/bigdatatypes/cassandra/SqlInstanceToCassandraSpec.scala b/cassandra/src/test/scala/org/datatools/bigdatatypes/cassandra/SqlInstanceToCassandraSpec.scala index be8afe9b..9e2bb78e 100644 --- a/cassandra/src/test/scala/org/datatools/bigdatatypes/cassandra/SqlInstanceToCassandraSpec.scala +++ b/cassandra/src/test/scala/org/datatools/bigdatatypes/cassandra/SqlInstanceToCassandraSpec.scala @@ -2,11 +2,14 @@ package org.datatools.bigdatatypes.cassandra import org.datatools.bigdatatypes.TestTypes.BasicTypes import org.datatools.bigdatatypes.basictypes.SqlType +import org.datatools.bigdatatypes.cassandra.CassandraTypeConversion.cassandraTupleType import org.datatools.bigdatatypes.formats.{DefaultFormats, Formats} import org.datatools.bigdatatypes.{CassandraTestTypes, UnitSpec} import org.datatools.bigdatatypes.cassandra.SqlInstanceToCassandra.* import org.datatools.bigdatatypes.conversions.SqlTypeConversion +import com.datastax.oss.driver.api.core.`type`.{DataType, DataTypes} + class SqlInstanceToCassandraSpec extends UnitSpec { implicit val defaultFormats: Formats = DefaultFormats @@ -27,4 +30,9 @@ class SqlInstanceToCassandraSpec extends UnitSpec { sql.asCassandra shouldBe CassandraTestTypes.basicFields } + "Tuple instance" should "be converted into Cassandra tuples" in { + val tuple: (String, DataType) = ("myLong", DataTypes.BIGINT) + SqlInstanceToCassandra[(String, DataType)].cassandraFields(tuple) shouldBe List(tuple) + } + } diff --git a/core/src/test/scala_2/org/datatools/bigdatatypes/formats/FormatsSpec.scala b/core/src/test/scala_2/org/datatools/bigdatatypes/formats/FormatsSpec.scala index b10c2724..fc1ecb0c 100644 --- a/core/src/test/scala_2/org/datatools/bigdatatypes/formats/FormatsSpec.scala +++ b/core/src/test/scala_2/org/datatools/bigdatatypes/formats/FormatsSpec.scala @@ -1,9 +1,47 @@ package org.datatools.bigdatatypes.formats import org.datatools.bigdatatypes.UnitSpec +import org.datatools.bigdatatypes.basictypes.SqlType._ +import org.datatools.bigdatatypes.basictypes.SqlTypeMode._ class FormatsSpec extends UnitSpec { - "Default Formats" should "do nothing to keys" in {} + behavior of "FormatsSpec" + + "Default Formats" should "leave keys unchanged" in { + DefaultFormats.transformKey("myValue", SqlString()) shouldBe "myValue" + DefaultFormats.transformKey("myValue", SqlInt()) shouldBe "myValue" + } + + "Default Formats" should "use BigDecimal precision 10 scale 0" in { + DefaultFormats.bigDecimal shouldBe DefaultFormats.BigDecimalPrecision(10, 0) + } + + "Base Formats" should "leave keys unchanged by default" in { + val formats = new Formats {} + formats.transformKey("myValue", SqlString()) shouldBe "myValue" + } + + "Snakify Formats" should "convert camelCase to snake_case" in { + SnakifyFormats.transformKey("myValue", SqlString()) shouldBe "my_value" + } + + "Snakify Formats" should "handle acronyms" in { + SnakifyFormats.transformKey("myURLValue", SqlString()) shouldBe "my_url_value" + } + + "KeyTypeExample Formats" should "prefix booleans with is_" in { + KeyTypeExampleFormats.transformKey("active", SqlBool()) shouldBe "is_active" + } + + "KeyTypeExample Formats" should "suffix dates and timestamps with _at" in { + KeyTypeExampleFormats.transformKey("birth", SqlDate()) shouldBe "birth_at" + KeyTypeExampleFormats.transformKey("created", SqlTimestamp()) shouldBe "created_at" + } + + "KeyTypeExample Formats" should "leave other types unchanged" in { + KeyTypeExampleFormats.transformKey("name", SqlString()) shouldBe "name" + KeyTypeExampleFormats.transformKey("count", SqlInt()) shouldBe "count" + } } diff --git a/spark/src/test/scala/org/datatools/bigdatatypes/spark/SparkTypeConversionSpec.scala b/spark/src/test/scala/org/datatools/bigdatatypes/spark/SparkTypeConversionSpec.scala index b07d0296..c30bfcda 100644 --- a/spark/src/test/scala/org/datatools/bigdatatypes/spark/SparkTypeConversionSpec.scala +++ b/spark/src/test/scala/org/datatools/bigdatatypes/spark/SparkTypeConversionSpec.scala @@ -9,6 +9,7 @@ import org.datatools.bigdatatypes.basictypes.SqlTypeMode.* import org.datatools.bigdatatypes.conversions.{SqlInstanceConversion, SqlTypeConversion} import org.datatools.bigdatatypes.formats.Formats.implicitDefaultFormats import org.datatools.bigdatatypes.spark.SparkTypeConversion.* +import org.datatools.bigdatatypes.spark.SqlInstanceToSpark.* /** Reverse conversion, from Spark types to [[SqlType]]s */ @@ -81,4 +82,29 @@ class SparkTypeConversionSpec extends UnitSpec { sqlType shouldBe extendedTypes } + "StructType instance" should "be converted into Spark fields" in { + val sf = StructField("myInt", IntegerType, nullable = false) + val sf2 = StructField("myString", StringType, nullable = true) + val st = StructType(List(sf, sf2)) + SqlInstanceToSpark[StructType].sparkFields(st) shouldBe SqlTypeToSpark.getSchema( + SqlInstanceConversion[StructType].getType(st) + ) + } + + "StructType instance" should "be converted using extension methods" in { + val sf = StructField("myInt", IntegerType, nullable = false) + val sf2 = StructField("myString", StringType, nullable = true) + val st = StructType(List(sf, sf2)) + st.asSparkFields shouldBe SqlInstanceToSpark[StructType].sparkFields(st) + st.asSparkSchema shouldBe StructType(SqlInstanceToSpark[StructType].sparkFields(st)) + } + + "SparkSchemas" should "build fields and schemas from instances" in { + val sf = StructField("myInt", IntegerType, nullable = false) + val sf2 = StructField("myString", StringType, nullable = true) + val st = StructType(List(sf, sf2)) + SparkSchemas.fields(st) shouldBe SqlInstanceToSpark[StructType].sparkFields(st) + SparkSchemas.schema(st) shouldBe StructType(SqlInstanceToSpark[StructType].sparkFields(st)) + } + }