From d26657ffa0aa5f98b260a430b947e25f45bf74fb Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Wed, 19 Aug 2026 20:35:48 +0000 Subject: [PATCH 1/3] impl(bigquery): use arrow format with jobs.query --- Cargo.lock | 46 + librarian.yaml | 8 +- src/bigquery/Cargo.toml | 1 + src/bigquery/src/datatypes.rs | 15 +- .../src/generated/complete_query_metadata.rs | 149 ++ src/bigquery/src/generated/query_metadata.rs | 151 ++ src/bigquery/src/generated/query_request.rs | 3 + src/bigquery/src/query/execution.rs | 13 +- src/bigquery/src/query/iterator.rs | 96 +- src/bigquery/src/query/query_handle.rs | 82 +- src/bigquery/src/query/row.rs | 443 +++++ src/bigquery/src/query/schema.rs | 119 +- src/generated/cloud/bigquery/v2/src/client.rs | 205 ++- src/generated/cloud/bigquery/v2/src/model.rs | 1563 ++++++++++++++++- .../cloud/bigquery/v2/src/model/debug.rs | 67 + .../bigquery/v2/src/model/deserialize.rs | 668 +++++++ .../cloud/bigquery/v2/src/model/serialize.rs | 224 +++ tests/bigquery/Cargo.toml | 2 +- 18 files changed, 3794 insertions(+), 61 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 291392b518..2934b0ad79 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -178,6 +178,8 @@ dependencies = [ "arrow-schema", "arrow-select", "flatbuffers", + "lz4_flex", + "zstd", ] [[package]] @@ -2281,6 +2283,7 @@ name = "google-cloud-bigquery" version = "0.16.0-preview" dependencies = [ "anyhow", + "arrow", "base64 0.23.1", "bytes", "google-cloud-auth", @@ -7616,6 +7619,15 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "lz4_flex" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226" +dependencies = [ + "twox-hash", +] + [[package]] name = "markdown" version = "1.0.0" @@ -10094,6 +10106,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "twox-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9" + [[package]] name = "typenum" version = "1.20.1" @@ -10666,3 +10684,31 @@ name = "zmij" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/librarian.yaml b/librarian.yaml index 24a1d27cca..76e43c2df4 100644 --- a/librarian.yaml +++ b/librarian.yaml @@ -22,8 +22,8 @@ sources: commit: 05b3ceb44e69212f6dad682f450862270749aee5 sha256: 11d07430eb3845b98a60178ee05e0656c0de6952d392e49a04f8d240e9d6a81f googleapis: - commit: 240b58fe7058f6fff77cba02f51d600170b6c421 - sha256: ca03f2c93b8f1e92fd75c87b2568992c96a2ecc37fde750c5bf83129fa17b407 + commit: 8856eed71a9cb2b9829d2617ee57fe2d8871a691 + sha256: 54104af35306b605a97f7d1b98b9b96f4a507cd18322b44d81f62eca17969b3a protobuf: commit: b407e8416e3893036aee5af9a12bd9b6a0e2b2e6 sha256: 55912546338433f465a552e9ef09930c63b9eb697053937416890cff83a8622d @@ -498,6 +498,8 @@ libraries: - .google.cloud.bigquery.v2.JobConfiguration.load - .google.cloud.bigquery.v2.JobConfiguration.extract - .google.cloud.bigquery.v2.QueryRequest.format_options + - .google.cloud.bigquery.v2.QueryRequest.query_results_format + - .google.cloud.bigquery.v2.QueryRequest.arrow_serialization_options - .google.cloud.bigquery.v2.GetQueryResultsRequest.format_options - .google.cloud.bigquery.v2.QueryRequest.request_id - .google.cloud.bigquery.v2.JobConfigurationQuery.continuous @@ -507,6 +509,8 @@ libraries: - .google.cloud.bigquery.v2.JobConfiguration.kind - .google.cloud.bigquery.v2.JobConfiguration.job_type - .google.cloud.bigquery.v2.QueryResponse.rows + - .google.cloud.bigquery.v2.QueryResponse.arrow_record_batch + - .google.cloud.bigquery.v2.QueryResponse.arrow_schema - .google.cloud.bigquery.v2.GetQueryResultsResponse.rows api_path: google/cloud/bigquery/v2 template: bigquery diff --git a/src/bigquery/Cargo.toml b/src/bigquery/Cargo.toml index 15842b8f8d..a44cd5adde 100644 --- a/src/bigquery/Cargo.toml +++ b/src/bigquery/Cargo.toml @@ -26,6 +26,7 @@ categories.workspace = true rust-version.workspace = true [dependencies] +arrow = { workspace = true, features = ["ipc", "ipc_compression"] } base64.workspace = true bytes.workspace = true http.workspace = true diff --git a/src/bigquery/src/datatypes.rs b/src/bigquery/src/datatypes.rs index c47fa4595b..8ab72a7698 100644 --- a/src/bigquery/src/datatypes.rs +++ b/src/bigquery/src/datatypes.rs @@ -227,9 +227,20 @@ impl FromSql for Range { Ok(Range { start, end }) } + wkt::Value::Object(obj) => { + let start = match obj.get("start") { + Some(wkt::Value::Null) | None => None, + Some(val) => Some(T::from_sql(val.clone())?), + }; + let end = match obj.get("end") { + Some(wkt::Value::Null) | None => None, + Some(val) => Some(T::from_sql(val.clone())?), + }; + Ok(Range { start, end }) + } wkt::Value::Null => Err(ConvertError::NotNull), other => Err(ConvertError::TypeMismatch { - expected: "string", + expected: "string or object", got: other, }), } @@ -283,7 +294,7 @@ mod tests { #[test_case(wkt::Value::String("[UNBOUNDED, 2026-05-29)".to_string()) => Ok(Range { start: None, end: Some(google_cloud_type::model::Date::new().set_year(2026).set_month(5).set_day(29)) }) ; "date range unbounded start")] #[test_case(wkt::Value::String("[UNBOUNDED, UNBOUNDED)".to_string()) => Ok(Range { start: None, end: None }) ; "date range unbounded both")] #[test_case(wkt::Value::Null => Err(TestConvertError::NotNull) ; "null range")] - #[test_case(wkt::Value::Number(123.into()) => Err(TestConvertError::TypeMismatch("string")) ; "range type mismatch")] + #[test_case(wkt::Value::Number(123.into()) => Err(TestConvertError::TypeMismatch("string or object")) ; "range type mismatch")] #[test_case(wkt::Value::String("[2026-05-28)".to_string()) => Err(TestConvertError::Convert("invalid range format: expected 2 parts, got 1".to_string())) ; "range invalid format one part")] #[test_case(wkt::Value::String("[2026-05-28, 2026-05-29, 2026-05-30)".to_string()) => Err(TestConvertError::Convert("invalid range format: expected 2 parts, got 3".to_string())) ; "range invalid format three parts")] #[test_case(wkt::Value::String("[".to_string()) => Err(TestConvertError::Convert("invalid range format: missing enclosing brackets".to_string())) ; "range too short")] diff --git a/src/bigquery/src/generated/complete_query_metadata.rs b/src/bigquery/src/generated/complete_query_metadata.rs index 8acad06266..f0d0f3953d 100644 --- a/src/bigquery/src/generated/complete_query_metadata.rs +++ b/src/bigquery/src/generated/complete_query_metadata.rs @@ -80,6 +80,12 @@ pub struct CompleteQueryMetadata { /// for DML statements INSERT, UPDATE or DELETE. pub num_dml_affected_rows: std::option::Option, + /// Output only. The number of rows out of `total_rows` returned in this + /// response. + /// + /// This feature is not yet available. + pub page_row_count: i64, + /// A token used for paging results. When this token is non-empty, it /// indicates additional results are available. pub page_token: std::string::String, @@ -99,6 +105,133 @@ pub struct CompleteQueryMetadata { /// state to either RUNNING or DONE. pub start_time: std::option::Option, + /// Output only. The type of query statement, if valid. + /// Possible values: + /// + /// * `SELECT`: + /// [`SELECT`](https://cloud.google.com/bigquery/docs/reference/standard-sql/query-syntax#select_list) + /// statement. + /// * `ASSERT`: + /// [`ASSERT`](https://cloud.google.com/bigquery/docs/reference/standard-sql/debugging-statements#assert) + /// statement. + /// * `INSERT`: + /// [`INSERT`](https://cloud.google.com/bigquery/docs/reference/standard-sql/dml-syntax#insert_statement) + /// statement. + /// * `UPDATE`: + /// [`UPDATE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/dml-syntax#update_statement) + /// statement. + /// * `DELETE`: + /// [`DELETE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-manipulation-language) + /// statement. + /// * `MERGE`: + /// [`MERGE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-manipulation-language) + /// statement. + /// * `CREATE_TABLE`: [`CREATE + /// TABLE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_table_statement) + /// statement, without `AS SELECT`. + /// * `CREATE_TABLE_AS_SELECT`: [`CREATE TABLE AS + /// SELECT`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_table_statement) + /// statement. + /// * `CREATE_VIEW`: [`CREATE + /// VIEW`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_view_statement) + /// statement. + /// * `CREATE_MODEL`: [`CREATE + /// MODEL`](https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-create#create_model_statement) + /// statement. + /// * `CREATE_MATERIALIZED_VIEW`: [`CREATE MATERIALIZED + /// VIEW`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_materialized_view_statement) + /// statement. + /// * `CREATE_FUNCTION`: [`CREATE + /// FUNCTION`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_function_statement) + /// statement. + /// * `CREATE_TABLE_FUNCTION`: [`CREATE TABLE + /// FUNCTION`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_table_function_statement) + /// statement. + /// * `CREATE_PROCEDURE`: [`CREATE + /// PROCEDURE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_procedure) + /// statement. + /// * `CREATE_ROW_ACCESS_POLICY`: [`CREATE ROW ACCESS + /// POLICY`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_row_access_policy_statement) + /// statement. + /// * `CREATE_SCHEMA`: [`CREATE + /// SCHEMA`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_schema_statement) + /// statement. + /// * `CREATE_SNAPSHOT_TABLE`: [`CREATE SNAPSHOT + /// TABLE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_snapshot_table_statement) + /// statement. + /// * `CREATE_SEARCH_INDEX`: [`CREATE SEARCH + /// INDEX`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_search_index_statement) + /// statement. + /// * `DROP_TABLE`: [`DROP + /// TABLE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#drop_table_statement) + /// statement. + /// * `DROP_EXTERNAL_TABLE`: [`DROP EXTERNAL + /// TABLE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#drop_external_table_statement) + /// statement. + /// * `DROP_VIEW`: [`DROP + /// VIEW`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#drop_view_statement) + /// statement. + /// * `DROP_MODEL`: [`DROP + /// MODEL`](https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-drop-model) + /// statement. + /// * `DROP_MATERIALIZED_VIEW`: [`DROP MATERIALIZED + /// VIEW`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#drop_materialized_view_statement) + /// statement. + /// * `DROP_FUNCTION` : [`DROP + /// FUNCTION`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#drop_function_statement) + /// statement. + /// * `DROP_TABLE_FUNCTION` : [`DROP TABLE + /// FUNCTION`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#drop_table_function) + /// statement. + /// * `DROP_PROCEDURE`: [`DROP + /// PROCEDURE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#drop_procedure_statement) + /// statement. + /// * `DROP_SEARCH_INDEX`: [`DROP SEARCH + /// INDEX`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#drop_search_index) + /// statement. + /// * `DROP_SCHEMA`: [`DROP + /// SCHEMA`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#drop_schema_statement) + /// statement. + /// * `DROP_SNAPSHOT_TABLE`: [`DROP SNAPSHOT + /// TABLE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#drop_snapshot_table_statement) + /// statement. + /// * `DROP_ROW_ACCESS_POLICY`: [`DROP [ALL] ROW ACCESS + /// POLICY|POLICIES`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#drop_row_access_policy_statement) + /// statement. + /// * `ALTER_TABLE`: [`ALTER + /// TABLE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#alter_table_set_options_statement) + /// statement. + /// * `ALTER_VIEW`: [`ALTER + /// VIEW`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#alter_view_set_options_statement) + /// statement. + /// * `ALTER_MATERIALIZED_VIEW`: [`ALTER MATERIALIZED + /// VIEW`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#alter_materialized_view_set_options_statement) + /// statement. + /// * `ALTER_SCHEMA`: [`ALTER + /// SCHEMA`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#alter_schema_set_options_statement) + /// statement. + /// * `SCRIPT`: + /// [`SCRIPT`](https://cloud.google.com/bigquery/docs/reference/standard-sql/procedural-language). + /// * `TRUNCATE_TABLE`: [`TRUNCATE + /// TABLE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/dml-syntax#truncate_table_statement) + /// statement. + /// * `CREATE_EXTERNAL_TABLE`: [`CREATE EXTERNAL + /// TABLE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_external_table_statement) + /// statement. + /// * `EXPORT_DATA`: [`EXPORT + /// DATA`](https://cloud.google.com/bigquery/docs/reference/standard-sql/other-statements#export_data_statement) + /// statement. + /// * `EXPORT_MODEL`: [`EXPORT + /// MODEL`](https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-export-model) + /// statement. + /// * `LOAD_DATA`: [`LOAD + /// DATA`](https://cloud.google.com/bigquery/docs/reference/standard-sql/other-statements#load_data_statement) + /// statement. + /// * `CALL`: + /// [`CALL`](https://cloud.google.com/bigquery/docs/reference/standard-sql/procedural-language#call) + /// statement. + pub statement_type: std::string::String, + /// Output only. If the project is configured to use on-demand pricing, /// then this field contains the total bytes billed for the job. /// If the project is configured to use flat-rate pricing, then you are @@ -298,6 +431,12 @@ impl CompleteQueryMetadata { self } + /// Sets the value of [page_row_count][crate::model::CompleteQueryMetadata::page_row_count]. + pub fn set_page_row_count>(mut self, v: T) -> Self { + self.page_row_count = v.into(); + self + } + /// Sets the value of [page_token][crate::model::CompleteQueryMetadata::page_token]. pub fn set_page_token>(mut self, v: T) -> Self { self.page_token = v.into(); @@ -364,6 +503,12 @@ impl CompleteQueryMetadata { self } + /// Sets the value of [statement_type][crate::model::CompleteQueryMetadata::statement_type]. + pub fn set_statement_type>(mut self, v: T) -> Self { + self.statement_type = v.into(); + self + } + /// Sets the value of [total_bytes_billed][crate::model::CompleteQueryMetadata::total_bytes_billed]. pub fn set_total_bytes_billed(mut self, v: T) -> Self where @@ -454,11 +599,13 @@ mod debug { debug_struct.field("kind", &self.kind); debug_struct.field("location", &self.location); debug_struct.field("num_dml_affected_rows", &self.num_dml_affected_rows); + debug_struct.field("page_row_count", &self.page_row_count); debug_struct.field("page_token", &self.page_token); debug_struct.field("query_id", &self.query_id); debug_struct.field("schema", &self.schema); debug_struct.field("session_info", &self.session_info); debug_struct.field("start_time", &self.start_time); + debug_struct.field("statement_type", &self.statement_type); debug_struct.field("total_bytes_billed", &self.total_bytes_billed); debug_struct.field("total_bytes_processed", &self.total_bytes_processed); debug_struct.field("total_rows", &self.total_rows); @@ -506,11 +653,13 @@ impl std::convert::From for Comp kind: resp.kind, location: resp.location, num_dml_affected_rows: resp.num_dml_affected_rows, + page_row_count: resp.page_row_count, page_token: resp.page_token, query_id: resp.query_id, schema: resp.schema, session_info: resp.session_info, start_time: resp.start_time, + statement_type: resp.statement_type, total_bytes_billed: resp.total_bytes_billed, total_bytes_processed: resp.total_bytes_processed, total_rows: resp.total_rows, diff --git a/src/bigquery/src/generated/query_metadata.rs b/src/bigquery/src/generated/query_metadata.rs index fccdb62dd2..47bd3b6e22 100644 --- a/src/bigquery/src/generated/query_metadata.rs +++ b/src/bigquery/src/generated/query_metadata.rs @@ -79,6 +79,12 @@ pub struct QueryMetadata { /// for DML statements INSERT, UPDATE or DELETE. pub num_dml_affected_rows: std::option::Option, + /// Output only. The number of rows out of `total_rows` returned in this + /// response. + /// + /// This feature is not yet available. + pub page_row_count: i64, + /// A token used for paging results. A non-empty token indicates that /// additional results are available. To see additional results, /// query the @@ -110,6 +116,133 @@ pub struct QueryMetadata { /// state to either RUNNING or DONE. pub start_time: std::option::Option, + /// Output only. The type of query statement, if valid. + /// Possible values: + /// + /// * `SELECT`: + /// [`SELECT`](https://cloud.google.com/bigquery/docs/reference/standard-sql/query-syntax#select_list) + /// statement. + /// * `ASSERT`: + /// [`ASSERT`](https://cloud.google.com/bigquery/docs/reference/standard-sql/debugging-statements#assert) + /// statement. + /// * `INSERT`: + /// [`INSERT`](https://cloud.google.com/bigquery/docs/reference/standard-sql/dml-syntax#insert_statement) + /// statement. + /// * `UPDATE`: + /// [`UPDATE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/dml-syntax#update_statement) + /// statement. + /// * `DELETE`: + /// [`DELETE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-manipulation-language) + /// statement. + /// * `MERGE`: + /// [`MERGE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-manipulation-language) + /// statement. + /// * `CREATE_TABLE`: [`CREATE + /// TABLE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_table_statement) + /// statement, without `AS SELECT`. + /// * `CREATE_TABLE_AS_SELECT`: [`CREATE TABLE AS + /// SELECT`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_table_statement) + /// statement. + /// * `CREATE_VIEW`: [`CREATE + /// VIEW`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_view_statement) + /// statement. + /// * `CREATE_MODEL`: [`CREATE + /// MODEL`](https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-create#create_model_statement) + /// statement. + /// * `CREATE_MATERIALIZED_VIEW`: [`CREATE MATERIALIZED + /// VIEW`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_materialized_view_statement) + /// statement. + /// * `CREATE_FUNCTION`: [`CREATE + /// FUNCTION`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_function_statement) + /// statement. + /// * `CREATE_TABLE_FUNCTION`: [`CREATE TABLE + /// FUNCTION`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_table_function_statement) + /// statement. + /// * `CREATE_PROCEDURE`: [`CREATE + /// PROCEDURE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_procedure) + /// statement. + /// * `CREATE_ROW_ACCESS_POLICY`: [`CREATE ROW ACCESS + /// POLICY`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_row_access_policy_statement) + /// statement. + /// * `CREATE_SCHEMA`: [`CREATE + /// SCHEMA`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_schema_statement) + /// statement. + /// * `CREATE_SNAPSHOT_TABLE`: [`CREATE SNAPSHOT + /// TABLE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_snapshot_table_statement) + /// statement. + /// * `CREATE_SEARCH_INDEX`: [`CREATE SEARCH + /// INDEX`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_search_index_statement) + /// statement. + /// * `DROP_TABLE`: [`DROP + /// TABLE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#drop_table_statement) + /// statement. + /// * `DROP_EXTERNAL_TABLE`: [`DROP EXTERNAL + /// TABLE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#drop_external_table_statement) + /// statement. + /// * `DROP_VIEW`: [`DROP + /// VIEW`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#drop_view_statement) + /// statement. + /// * `DROP_MODEL`: [`DROP + /// MODEL`](https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-drop-model) + /// statement. + /// * `DROP_MATERIALIZED_VIEW`: [`DROP MATERIALIZED + /// VIEW`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#drop_materialized_view_statement) + /// statement. + /// * `DROP_FUNCTION` : [`DROP + /// FUNCTION`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#drop_function_statement) + /// statement. + /// * `DROP_TABLE_FUNCTION` : [`DROP TABLE + /// FUNCTION`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#drop_table_function) + /// statement. + /// * `DROP_PROCEDURE`: [`DROP + /// PROCEDURE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#drop_procedure_statement) + /// statement. + /// * `DROP_SEARCH_INDEX`: [`DROP SEARCH + /// INDEX`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#drop_search_index) + /// statement. + /// * `DROP_SCHEMA`: [`DROP + /// SCHEMA`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#drop_schema_statement) + /// statement. + /// * `DROP_SNAPSHOT_TABLE`: [`DROP SNAPSHOT + /// TABLE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#drop_snapshot_table_statement) + /// statement. + /// * `DROP_ROW_ACCESS_POLICY`: [`DROP [ALL] ROW ACCESS + /// POLICY|POLICIES`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#drop_row_access_policy_statement) + /// statement. + /// * `ALTER_TABLE`: [`ALTER + /// TABLE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#alter_table_set_options_statement) + /// statement. + /// * `ALTER_VIEW`: [`ALTER + /// VIEW`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#alter_view_set_options_statement) + /// statement. + /// * `ALTER_MATERIALIZED_VIEW`: [`ALTER MATERIALIZED + /// VIEW`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#alter_materialized_view_set_options_statement) + /// statement. + /// * `ALTER_SCHEMA`: [`ALTER + /// SCHEMA`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#alter_schema_set_options_statement) + /// statement. + /// * `SCRIPT`: + /// [`SCRIPT`](https://cloud.google.com/bigquery/docs/reference/standard-sql/procedural-language). + /// * `TRUNCATE_TABLE`: [`TRUNCATE + /// TABLE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/dml-syntax#truncate_table_statement) + /// statement. + /// * `CREATE_EXTERNAL_TABLE`: [`CREATE EXTERNAL + /// TABLE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_external_table_statement) + /// statement. + /// * `EXPORT_DATA`: [`EXPORT + /// DATA`](https://cloud.google.com/bigquery/docs/reference/standard-sql/other-statements#export_data_statement) + /// statement. + /// * `EXPORT_MODEL`: [`EXPORT + /// MODEL`](https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-export-model) + /// statement. + /// * `LOAD_DATA`: [`LOAD + /// DATA`](https://cloud.google.com/bigquery/docs/reference/standard-sql/other-statements#load_data_statement) + /// statement. + /// * `CALL`: + /// [`CALL`](https://cloud.google.com/bigquery/docs/reference/standard-sql/procedural-language#call) + /// statement. + pub statement_type: std::string::String, + /// Output only. Information about the job, including starting time and ending /// time of the job. pub statistics: std::option::Option, @@ -345,6 +478,12 @@ impl QueryMetadata { self } + /// Sets the value of [page_row_count][crate::model::QueryMetadata::page_row_count]. + pub fn set_page_row_count>(mut self, v: T) -> Self { + self.page_row_count = v.into(); + self + } + /// Sets the value of [page_token][crate::model::QueryMetadata::page_token]. pub fn set_page_token>(mut self, v: T) -> Self { self.page_token = v.into(); @@ -426,6 +565,12 @@ impl QueryMetadata { self } + /// Sets the value of [statement_type][crate::model::QueryMetadata::statement_type]. + pub fn set_statement_type>(mut self, v: T) -> Self { + self.statement_type = v.into(); + self + } + /// Sets the value of [statistics][crate::model::QueryMetadata::statistics]. pub fn set_statistics(mut self, v: T) -> Self where @@ -560,6 +705,7 @@ mod debug { debug_struct.field("kind", &self.kind); debug_struct.field("location", &self.location); debug_struct.field("num_dml_affected_rows", &self.num_dml_affected_rows); + debug_struct.field("page_row_count", &self.page_row_count); debug_struct.field("page_token", &self.page_token); debug_struct.field("principal_subject", &self.principal_subject); debug_struct.field("query_id", &self.query_id); @@ -567,6 +713,7 @@ mod debug { debug_struct.field("self_link", &self.self_link); debug_struct.field("session_info", &self.session_info); debug_struct.field("start_time", &self.start_time); + debug_struct.field("statement_type", &self.statement_type); debug_struct.field("statistics", &self.statistics); debug_struct.field("status", &self.status); debug_struct.field("total_bytes_billed", &self.total_bytes_billed); @@ -615,11 +762,13 @@ impl std::convert::From for Quer kind: resp.kind, location: resp.location, num_dml_affected_rows: resp.num_dml_affected_rows, + page_row_count: resp.page_row_count, page_token: resp.page_token, query_id: resp.query_id, schema: resp.schema, session_info: resp.session_info, start_time: resp.start_time, + statement_type: resp.statement_type, total_bytes_billed: resp.total_bytes_billed, total_bytes_processed: resp.total_bytes_processed, total_rows: resp.total_rows, @@ -643,11 +792,13 @@ impl std::convert::From for crate::generated::CompleteQueryMetada kind: md.kind, location: md.location, num_dml_affected_rows: md.num_dml_affected_rows, + page_row_count: md.page_row_count, page_token: md.page_token, query_id: md.query_id, schema: md.schema, session_info: md.session_info, start_time: md.start_time, + statement_type: md.statement_type, total_bytes_billed: md.total_bytes_billed, total_bytes_processed: md.total_bytes_processed, total_rows: md.total_rows, diff --git a/src/bigquery/src/generated/query_request.rs b/src/bigquery/src/generated/query_request.rs index 5f55ee7703..fe1aa09de0 100644 --- a/src/bigquery/src/generated/query_request.rs +++ b/src/bigquery/src/generated/query_request.rs @@ -165,6 +165,9 @@ pub struct QueryRequest { /// Optional. The reservation that jobs.query request would use. User can /// specify a reservation to execute the job.query. The expected format is /// `projects/{project}/locations/{location}/reservations/{reservation}`. + /// Forces the query to use on-demand billing when set to `none`. + /// This requires the project or organization to have + /// `reservation_override_mode` set to `ALLOW_ANY_OVERRIDE`. pub reservation: std::option::Option, /// Allows the schema of the destination table to be updated as a side effect diff --git a/src/bigquery/src/query/execution.rs b/src/bigquery/src/query/execution.rs index a167884103..a9f8e04fa6 100644 --- a/src/bigquery/src/query/execution.rs +++ b/src/bigquery/src/query/execution.rs @@ -192,9 +192,18 @@ impl RetryContext { let query_request: QueryRequest = self.template.request.clone().into(); let query_request = query_request .set_format_options( - google_cloud_bigquery_v2::model::DataFormatOptions::new() + crate::model::DataFormatOptions::new() .set_use_int64_timestamp(true), ) + .set_query_results_format(crate::model::query_request::QueryResultsFormat::Arrow) + .set_results_format_serialization_options( + crate::model::query_request::ResultsFormatSerializationOptions::ArrowSerializationOptions( + Box::new(crate::model::ArrowSerializationOptions::new() + .set_buffer_compression( + crate::model::arrow_serialization_options::CompressionCodec::Zstd, + )) + ) + ) .set_request_id(query_request_id); let req = PostQueryRequest::new() .set_project_id(project_id) @@ -254,7 +263,7 @@ mod tests { .clone() .expect("should have job_ref"); assert_eq!(job_ref.job_id, "my-job-123", "{job_ref:?}"); - assert!(query.cached_rows.is_some(), "{query:?}"); + assert!(query.cached_data.is_some(), "{query:?}"); Ok(()) } diff --git a/src/bigquery/src/query/iterator.rs b/src/bigquery/src/query/iterator.rs index 1a6a20fb2f..76014bd114 100644 --- a/src/bigquery/src/query/iterator.rs +++ b/src/bigquery/src/query/iterator.rs @@ -13,10 +13,14 @@ // limitations under the License. use crate::error::RowError; +use crate::query::query_handle::CachedData; use crate::query::{CompleteQuery, Row, Schema}; +use arrow::ipc::reader::StreamReader; +use arrow::record_batch::RecordBatch; use google_cloud_bigquery_v2::client::JobService; use google_cloud_bigquery_v2::model::{GetQueryResultsRequest, JobReference}; use std::collections::VecDeque; +use std::io::{Cursor, Read}; use std::sync::Arc; pub type Result = std::result::Result; @@ -53,18 +57,40 @@ pub struct RowIterator { job_ref: Option, schema: Arc, page_token: Option, + record_batches: VecDeque, + row_index: usize, rows: VecDeque, max_results: Option, } impl RowIterator { pub(crate) fn new(q: CompleteQuery) -> Self { + let (rows, record_batches) = match q.cached_data { + CachedData::Rows(rows) => (rows, VecDeque::new()), + CachedData::Arrow { + serialized_record_batch, + serialized_schema, + } => { + let reader = StreamReader::try_new( + Cursor::new(serialized_schema).chain(Cursor::new(serialized_record_batch)), + None, + ) + .expect("valid arrow IPC stream"); // TODO: convert error + let batches = reader + .collect::, _>>() + .expect("valid record batches"); // TODO: convert error + (VecDeque::new(), batches) + } + }; + Self { job_service: q.job_service, job_ref: q.job_ref, schema: q.schema, page_token: q.page_token, - rows: q.cached_rows, + record_batches, + row_index: 0, + rows, max_results: q.max_results, } } @@ -111,6 +137,16 @@ impl RowIterator { /// ``` pub async fn next(&mut self) -> Option> { loop { + while let Some(batch) = self.record_batches.front() { + if self.row_index < batch.num_rows() { + let idx = self.row_index; + self.row_index += 1; + return Some(Row::try_new_from_arrow(batch, idx, &self.schema)); + } + self.record_batches.pop_front(); + self.row_index = 0; + } + if let Some(raw_row) = self.rows.pop_front() { return Some(Row::try_new(raw_row, &self.schema)); } @@ -119,7 +155,7 @@ impl RowIterator { return Some(Err(e)); } - if self.rows.is_empty() && self.page_token.is_none() { + if self.record_batches.is_empty() && self.rows.is_empty() && self.page_token.is_none() { return None; } } @@ -417,4 +453,60 @@ mod tests { ); Ok(()) } + + #[tokio::test] + async fn test_row_iterator_cached_arrow() -> TestResult { + use arrow::array::{Int64Array, StringArray}; + use arrow::datatypes::{DataType, Field, Schema as ArrowSchema}; + use arrow::ipc::writer::StreamWriter; + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("col", DataType::Utf8, false), + Field::new("num", DataType::Int64, false), + ])); + + let mut schema_buf = Vec::new(); + let _ = StreamWriter::try_new(&mut schema_buf, &arrow_schema)?; + + let col = StringArray::from(vec!["hello", "world"]); + let num = Int64Array::from(vec![42, 100]); + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new(col), Arc::new(num)])?; + + let mut batch_buf = Vec::new(); + let mut writer = StreamWriter::try_new(&mut batch_buf, &arrow_schema)?; + writer.write(&batch)?; + let batch_buf = batch_buf[schema_buf.len()..].to_vec(); + + let table_schema = TableSchema::new().set_fields([ + TableFieldSchema::new().set_name("col").set_type("STRING"), + TableFieldSchema::new().set_name("num").set_type("INTEGER"), + ]); + let schema = Arc::new(Schema::new(table_schema)); + + let job_service = create_job_service(MockJobService::new()); + let q = CompleteQuery { + job_service, + job_ref: None, + cached_data: CachedData::Arrow { + serialized_schema: schema_buf.into(), + serialized_record_batch: batch_buf.into(), + }, + schema, + page_token: None, + metadata: crate::generated::CompleteQueryMetadata::default(), + max_results: None, + }; + + let mut iter = q.read(); + let row1 = iter.next().await.expect("row 1")?; + assert_eq!(row1.get::("col"), "hello"); + assert_eq!(row1.get::("num"), 42); + + let row2 = iter.next().await.expect("row 2")?; + assert_eq!(row2.get::("col"), "world"); + assert_eq!(row2.get::("num"), 100); + + assert!(iter.next().await.is_none()); + Ok(()) + } } diff --git a/src/bigquery/src/query/query_handle.rs b/src/bigquery/src/query/query_handle.rs index 1a0c0d76ec..21f7a98693 100644 --- a/src/bigquery/src/query/query_handle.rs +++ b/src/bigquery/src/query/query_handle.rs @@ -14,12 +14,14 @@ use crate::error::QueryError; use crate::generated::{CompleteQueryMetadata, QueryMetadata}; +use crate::model::query_response::{Results, ResultsSchema}; use crate::model::{ GetQueryResultsRequest, GetQueryResultsResponse, Job, JobReference, QueryResponse, }; use crate::query::execution::RetryContext; use crate::query::{Result, RowIterator, Schema}; use crate::retry_policy::JobRetryResult; +use bytes::Bytes; use google_cloud_bigquery_v2::builder::job_service::GetJob; use google_cloud_bigquery_v2::client::JobService; use google_cloud_gax::exponential_backoff::ExponentialBackoffBuilder; @@ -56,11 +58,20 @@ pub struct Query { pub(crate) job_service: Arc, pub(crate) completed: bool, pub(crate) metadata: QueryMetadata, - pub(crate) cached_rows: Option>, + pub(crate) cached_data: Option, pub(crate) max_results: Option, pub(crate) retry_context: Option, } +#[derive(Clone, Debug)] +pub(crate) enum CachedData { + Rows(VecDeque), + Arrow { + serialized_record_batch: Bytes, + serialized_schema: Bytes, + }, +} + impl Query { pub(crate) fn from_job( job_service: Arc, @@ -76,7 +87,7 @@ impl Query { Self { job_service, completed, - cached_rows: None, + cached_data: None, metadata: QueryMetadata::from(initial_job), retry_context, max_results, @@ -90,12 +101,26 @@ impl Query { max_results: Option, ) -> Self { let completed = query_response.job_complete.unwrap_or(false); - let cached_rows = VecDeque::from(std::mem::take(&mut query_response.rows)); + let cached_data = if let ( + Some(ResultsSchema::ArrowSchema(schema)), + Some(Results::ArrowRecordBatch(results)), + ) = ( + query_response.results_schema.take(), + query_response.results.take(), + ) { + Some(CachedData::Arrow { + serialized_record_batch: results.serialized_record_batch, + serialized_schema: schema.serialized_schema, + }) + } else { + let cached_rows = VecDeque::from(std::mem::take(&mut query_response.rows)); + Some(CachedData::Rows(cached_rows)) + }; let metadata = QueryMetadata::from(query_response); Self { job_service, completed, - cached_rows: Some(cached_rows), + cached_data, metadata, retry_context, max_results, @@ -200,16 +225,16 @@ impl Query { job_service, completed, metadata, - cached_rows, + cached_data, max_results, retry_context, } = self; - if let (true, Some(cached_rows)) = (completed, cached_rows) { + if let (true, Some(cached_data)) = (completed, cached_data) { return Ok(CompleteQuery::from_query_metadata( job_service, metadata, - cached_rows, + cached_data, max_results, )); } @@ -280,7 +305,7 @@ impl Query { pub struct CompleteQuery { pub(crate) job_service: Arc, pub(crate) job_ref: Option, - pub(crate) cached_rows: VecDeque, + pub(crate) cached_data: CachedData, pub(crate) schema: Arc, pub(crate) page_token: Option, pub(crate) metadata: CompleteQueryMetadata, @@ -307,7 +332,7 @@ impl CompleteQuery { Self { job_service, job_ref: Some(job_ref.clone()), - cached_rows, + cached_data: CachedData::Rows(cached_rows), page_token, schema, metadata, @@ -318,14 +343,31 @@ impl CompleteQuery { pub(crate) fn from_query_metadata( job_service: Arc, metadata: QueryMetadata, - cached_rows: VecDeque, + cached_data: CachedData, max_results: Option, ) -> Self { let job_ref = metadata.job_reference.clone(); let metadata = CompleteQueryMetadata::from(metadata); - // DDL/DML queries have no schema. - let schema = metadata.schema.clone().unwrap_or_default(); - let schema = Arc::new(Schema::new(schema)); + let schema = match &cached_data { + CachedData::Rows(_) => { + // DDL/DML queries have no schema. + let schema = metadata.schema.clone().unwrap_or_default(); + Arc::new(Schema::new(schema)) + } + CachedData::Arrow { + serialized_schema, .. + } => { + match Schema::try_from_arrow_ipc(serialized_schema) { + Ok(s) => Arc::new(s), + Err(_) => { + // DDL/DML queries have no schema. + let schema = metadata.schema.clone().unwrap_or_default(); + Arc::new(Schema::new(schema)) + } + } + } + }; + let page_token = if metadata.page_token.is_empty() { None } else { @@ -334,7 +376,7 @@ impl CompleteQuery { Self { job_service, job_ref, - cached_rows, + cached_data, page_token, schema, metadata, @@ -508,7 +550,7 @@ mod tests { mut query_res: QueryResponse, max_results: Option, ) -> Self { - let cached_rows = std::mem::take(&mut query_res.rows).into(); + let cached_rows = CachedData::Rows(VecDeque::from(std::mem::take(&mut query_res.rows))); let metadata = QueryMetadata::from(query_res); Self::from_query_metadata(job_service, metadata, cached_rows, max_results) } @@ -533,7 +575,10 @@ mod tests { let completed = query.until_done().await?; assert_eq!(completed.job_ref.as_ref().unwrap().job_id, "some_job_id"); assert_eq!(completed.page_token, Some("some_page_token".to_string())); - assert_eq!(completed.cached_rows.len(), 1); + match &completed.cached_data { + CachedData::Rows(rows) => assert_eq!(rows.len(), 1), + _ => panic!("expected rows"), + } let metadata = completed.metadata(); assert_eq!(metadata.cache_hit, Some(true)); @@ -595,7 +640,10 @@ mod tests { let completed = query.until_done().await?; assert_eq!(completed.job_ref.as_ref().unwrap().job_id, "some_job_id"); assert_eq!(completed.page_token, None); - assert_eq!(completed.cached_rows.len(), 2); + match &completed.cached_data { + CachedData::Rows(rows) => assert_eq!(rows.len(), 2), + _ => panic!("expected rows"), + } let metadata = completed.metadata(); assert_eq!(metadata.cache_hit, Some(false)); diff --git a/src/bigquery/src/query/row.rs b/src/bigquery/src/query/row.rs index 0c43cfd98b..647a7cc0c6 100644 --- a/src/bigquery/src/query/row.rs +++ b/src/bigquery/src/query/row.rs @@ -14,6 +14,17 @@ use crate::error::{ConvertError, RowError}; use crate::query::{FromSql, Schema}; +use arrow::array::{ + Array, BinaryArray, BooleanArray, Float32Array, Float64Array, Int8Array, Int16Array, + Int32Array, Int64Array, IntervalMonthDayNanoArray, LargeBinaryArray, LargeListArray, + LargeStringArray, ListArray, StringArray, StructArray, TimestampMicrosecondArray, + TimestampMillisecondArray, TimestampNanosecondArray, TimestampSecondArray, UInt8Array, + UInt16Array, UInt32Array, UInt64Array, +}; +use arrow::datatypes::IntervalUnit; +use arrow::datatypes::{DataType, TimeUnit}; +use arrow::record_batch::RecordBatch; +use base64::Engine; use std::sync::Arc; use wkt::{ListValue, Struct, Value}; @@ -133,6 +144,32 @@ impl Row { }) } + pub(crate) fn try_new_from_arrow( + batch: &RecordBatch, + row_idx: usize, + schema: &Arc, + ) -> Result { + if batch.num_columns() != schema.len() { + return Err(RowError::InvalidRowFormat(format!( + "schema and row cell mismatch (expected {}, got {})", + schema.len(), + batch.num_columns() + ))); + } + + let mut values = ListValue::new(); + for col_idx in 0..batch.num_columns() { + let col = batch.column(col_idx); + let value = arrow_to_value(col.as_ref(), row_idx)?; + values.push(value); + } + + Ok(Self { + values: Value::Array(values), + schema: schema.clone(), + }) + } + fn resolve_index(&self, col: &I) -> Result { col.index(self) .ok_or_else(|| RowError::ColumnNotFound(format!("{col}"))) @@ -352,6 +389,262 @@ fn convert_basic_type(value: String, field_name: &str, field_type: &str) -> Resu } } +fn arrow_to_value(array: &dyn Array, row_idx: usize) -> Result { + if array.is_null(row_idx) { + return Ok(Value::Null); + } + + match array.data_type() { + DataType::Null => Ok(Value::Null), + DataType::Boolean => { + let arr = array + .as_any() + .downcast_ref::() + .ok_or_else(|| RowError::InvalidRowFormat("expected BooleanArray".into()))?; + Ok(Value::Bool(arr.value(row_idx))) + } + DataType::Int8 => { + let arr = array + .as_any() + .downcast_ref::() + .ok_or_else(|| RowError::InvalidRowFormat("expected Int8Array".into()))?; + Ok(Value::Number(serde_json::Number::from(arr.value(row_idx)))) + } + DataType::Int16 => { + let arr = array + .as_any() + .downcast_ref::() + .ok_or_else(|| RowError::InvalidRowFormat("expected Int16Array".into()))?; + Ok(Value::Number(serde_json::Number::from(arr.value(row_idx)))) + } + DataType::Int32 => { + let arr = array + .as_any() + .downcast_ref::() + .ok_or_else(|| RowError::InvalidRowFormat("expected Int32Array".into()))?; + Ok(Value::Number(serde_json::Number::from(arr.value(row_idx)))) + } + DataType::Int64 => { + let arr = array + .as_any() + .downcast_ref::() + .ok_or_else(|| RowError::InvalidRowFormat("expected Int64Array".into()))?; + Ok(Value::Number(serde_json::Number::from(arr.value(row_idx)))) + } + DataType::UInt8 => { + let arr = array + .as_any() + .downcast_ref::() + .ok_or_else(|| RowError::InvalidRowFormat("expected UInt8Array".into()))?; + Ok(Value::Number(serde_json::Number::from(arr.value(row_idx)))) + } + DataType::UInt16 => { + let arr = array + .as_any() + .downcast_ref::() + .ok_or_else(|| RowError::InvalidRowFormat("expected UInt16Array".into()))?; + Ok(Value::Number(serde_json::Number::from(arr.value(row_idx)))) + } + DataType::UInt32 => { + let arr = array + .as_any() + .downcast_ref::() + .ok_or_else(|| RowError::InvalidRowFormat("expected UInt32Array".into()))?; + Ok(Value::Number(serde_json::Number::from(arr.value(row_idx)))) + } + DataType::UInt64 => { + let arr = array + .as_any() + .downcast_ref::() + .ok_or_else(|| RowError::InvalidRowFormat("expected UInt64Array".into()))?; + Ok(Value::Number(serde_json::Number::from(arr.value(row_idx)))) + } + DataType::Float32 => { + let arr = array + .as_any() + .downcast_ref::() + .ok_or_else(|| RowError::InvalidRowFormat("expected Float32Array".into()))?; + let n = serde_json::Number::from_f64(arr.value(row_idx) as f64) + .ok_or_else(|| RowError::InvalidRowFormat("invalid f32 value".into()))?; + Ok(Value::Number(n)) + } + DataType::Float64 => { + let arr = array + .as_any() + .downcast_ref::() + .ok_or_else(|| RowError::InvalidRowFormat("expected Float64Array".into()))?; + let n = serde_json::Number::from_f64(arr.value(row_idx)) + .ok_or_else(|| RowError::InvalidRowFormat("invalid f64 value".into()))?; + Ok(Value::Number(n)) + } + DataType::Utf8 => { + let arr = array + .as_any() + .downcast_ref::() + .ok_or_else(|| RowError::InvalidRowFormat("expected StringArray".into()))?; + Ok(Value::String(arr.value(row_idx).to_string())) + } + DataType::LargeUtf8 => { + let arr = array + .as_any() + .downcast_ref::() + .ok_or_else(|| RowError::InvalidRowFormat("expected LargeStringArray".into()))?; + Ok(Value::String(arr.value(row_idx).to_string())) + } + DataType::Binary => { + let arr = array + .as_any() + .downcast_ref::() + .ok_or_else(|| RowError::InvalidRowFormat("expected BinaryArray".into()))?; + Ok(Value::String( + base64::prelude::BASE64_STANDARD.encode(arr.value(row_idx)), + )) + } + DataType::LargeBinary => { + let arr = array + .as_any() + .downcast_ref::() + .ok_or_else(|| RowError::InvalidRowFormat("expected LargeBinaryArray".into()))?; + Ok(Value::String( + base64::prelude::BASE64_STANDARD.encode(arr.value(row_idx)), + )) + } + DataType::Interval(unit) => convert_arrow_interval(array, row_idx, unit), + DataType::Timestamp(unit, Some(_)) => convert_arrow_timestamp(array, row_idx, unit), + DataType::Struct(_) => convert_arrow_struct(array, row_idx), + DataType::List(_) | DataType::LargeList(_) => convert_arrow_list(array, row_idx), + _ => { + let formatter = + arrow::util::display::ArrayFormatter::try_new(array, &Default::default()).map_err( + |e| RowError::InvalidRowFormat(format!("failed to format arrow value: {e}")), + )?; + Ok(Value::String(formatter.value(row_idx).to_string())) + } + } +} + +fn convert_arrow_interval(array: &dyn Array, row_idx: usize, unit: &IntervalUnit) -> Result { + match unit { + IntervalUnit::MonthDayNano => { + let arr = array + .as_any() + .downcast_ref::() + .ok_or_else(|| RowError::InvalidRowFormat("expected IntervalArray".into()))?; + let v = arr.value(row_idx); + // Format Year-Month (e.g. "1-2" or "-1-2") + let ym_sign = if v.months < 0 { "-" } else { "" }; + let years = v.months.abs() / 12; + let months = v.months.abs() % 12; + + // Format Time H:MM:SS[.fffffffff] + let (time_sign, total_nanos) = if v.nanoseconds < 0 { + ("-", (-v.nanoseconds) as u64) + } else { + ("", v.nanoseconds as u64) + }; + let nanos = total_nanos % 1_000_000_000; + let total_secs = total_nanos / 1_000_000_000; + let seconds = total_secs % 60; + let total_mins = total_secs / 60; + let minutes = total_mins % 60; + let hours = total_mins / 60; + + let time_str = if nanos == 0 { + format!("{time_sign}{hours}:{minutes:02}:{seconds:02}") + } else { + let frac = format!("{nanos:09}"); + let frac = frac.trim_end_matches('0'); + format!("{time_sign}{hours}:{minutes:02}:{seconds:02}.{frac}") + }; + + Ok(Value::String(format!( + "{ym_sign}{years}-{months} {} {time_str}", + v.days + ))) + } + _ => Err(RowError::InvalidRowFormat(format!( + "unsupported interval unit: {unit:?}" + ))), + } +} + +fn convert_arrow_timestamp(array: &dyn Array, row_idx: usize, unit: &TimeUnit) -> Result { + let micros = match unit { + TimeUnit::Microsecond => { + let arr = array + .as_any() + .downcast_ref::() + .ok_or_else(|| { + RowError::InvalidRowFormat("expected TimestampMicrosecondArray".into()) + })?; + arr.value(row_idx) + } + TimeUnit::Millisecond => { + let arr = array + .as_any() + .downcast_ref::() + .ok_or_else(|| { + RowError::InvalidRowFormat("expected TimestampMillisecondArray".into()) + })?; + arr.value(row_idx) * 1_000 + } + TimeUnit::Second => { + let arr = array + .as_any() + .downcast_ref::() + .ok_or_else(|| { + RowError::InvalidRowFormat("expected TimestampSecondArray".into()) + })?; + arr.value(row_idx) * 1_000_000 + } + TimeUnit::Nanosecond => { + let arr = array + .as_any() + .downcast_ref::() + .ok_or_else(|| { + RowError::InvalidRowFormat("expected TimestampNanosecondArray".into()) + })?; + arr.value(row_idx) / 1_000 + } + }; + Ok(Value::Number(serde_json::Number::from(micros))) +} + +fn convert_arrow_struct(array: &dyn Array, row_idx: usize) -> Result { + let struct_arr = array + .as_any() + .downcast_ref::() + .ok_or_else(|| RowError::InvalidRowFormat("expected StructArray".into()))?; + let mut obj = Struct::new(); + for (field, col) in struct_arr.fields().iter().zip(struct_arr.columns()) { + let val = arrow_to_value(col.as_ref(), row_idx)?; + obj.insert(field.name().to_string(), val); + } + Ok(Value::Object(obj)) +} + +fn convert_arrow_list(array: &dyn Array, row_idx: usize) -> Result { + if let Some(list_arr) = array.as_any().downcast_ref::() { + let sub_arr = list_arr.value(row_idx); + let mut values = ListValue::new(); + for i in 0..sub_arr.len() { + values.push(arrow_to_value(sub_arr.as_ref(), i)?); + } + return Ok(Value::Array(values)); + } + if let Some(list_arr) = array.as_any().downcast_ref::() { + let sub_arr = list_arr.value(row_idx); + let mut values = ListValue::new(); + for i in 0..sub_arr.len() { + values.push(arrow_to_value(sub_arr.as_ref(), i)?); + } + return Ok(Value::Array(values)); + } + Err(RowError::InvalidRowFormat( + "expected ListArray or LargeListArray".into(), + )) +} + #[cfg(test)] mod tests { use super::*; @@ -837,4 +1130,154 @@ mod tests { assert!(matches!(err, RowError::ColumnNotFound(col) if col == "custom_int")); Ok(()) } + + #[test] + fn try_new_from_arrow_batch() -> TestResult { + use arrow::array::{ + BooleanArray, Float64Array, Int64Array, StringArray, TimestampMicrosecondArray, + }; + use arrow::datatypes::{DataType, Field, Schema as ArrowSchema, TimeUnit}; + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("name", DataType::Utf8, false), + Field::new("age", DataType::Int64, true), + Field::new("active", DataType::Boolean, false), + Field::new("score", DataType::Float64, false), + Field::new( + "created_ts", + DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())), + false, + ), + Field::new( + "created_dt", + DataType::Timestamp(TimeUnit::Microsecond, None), + false, + ), + ])); + + let name = StringArray::from(vec!["Alice", "Bob"]); + let age = Int64Array::from(vec![Some(30), None]); + let active = BooleanArray::from(vec![true, false]); + let score = Float64Array::from(vec![98.5, 87.25]); + let created_ts = + TimestampMicrosecondArray::from(vec![1_600_000_000_000_000, 1_700_000_000_000_000]) + .with_timezone("UTC"); + let created_dt = + TimestampMicrosecondArray::from(vec![1_600_000_000_000_000, 1_700_000_000_000_000]); + + let batch = RecordBatch::try_new( + arrow_schema, + vec![ + Arc::new(name), + Arc::new(age), + Arc::new(active), + Arc::new(score), + Arc::new(created_ts), + Arc::new(created_dt), + ], + )?; + + let table_schema = TableSchema::new().set_fields([ + TableFieldSchema::new().set_name("name").set_type("STRING"), + TableFieldSchema::new().set_name("age").set_type("INTEGER"), + TableFieldSchema::new() + .set_name("active") + .set_type("BOOLEAN"), + TableFieldSchema::new().set_name("score").set_type("FLOAT"), + TableFieldSchema::new() + .set_name("created_ts") + .set_type("TIMESTAMP"), + TableFieldSchema::new() + .set_name("created_dt") + .set_type("DATETIME"), + ]); + let schema = Arc::new(Schema::new(table_schema)); + + let row0 = Row::try_new_from_arrow(&batch, 0, &schema)?; + assert_eq!(row0.get::("name"), "Alice"); + assert_eq!(row0.get::, _>("age"), Some(30)); + assert!(row0.get::("active")); + assert_eq!(row0.get::("score"), 98.5); + assert_eq!( + row0.get::("created_ts"), + wkt::Timestamp::new(1_600_000_000, 0).unwrap() + ); + + let row1 = Row::try_new_from_arrow(&batch, 1, &schema)?; + assert_eq!(row1.get::("name"), "Bob"); + assert_eq!(row1.get::, _>("age"), None); + assert!(!row1.get::("active")); + assert_eq!(row1.get::("score"), 87.25); + assert_eq!( + row1.get::("created_ts"), + wkt::Timestamp::new(1_700_000_000, 0).unwrap() + ); + + Ok(()) + } + + #[test] + fn try_new_from_arrow_interval() -> TestResult { + use crate::datatypes::Interval; + use arrow::array::IntervalMonthDayNanoArray; + use arrow::datatypes::{DataType, Field, IntervalUnit, Schema as ArrowSchema}; + + let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new( + "duration", + DataType::Interval(IntervalUnit::MonthDayNano), + false, + )])); + + let intervals = IntervalMonthDayNanoArray::from(vec![ + arrow::datatypes::IntervalMonthDayNanoType::make_value( + 14, + 3, + (4 * 3600 + 5 * 60 + 6) * 1_000_000_000 + 789_123_456, + ), + arrow::datatypes::IntervalMonthDayNanoType::make_value( + -14, + -3, + -((4 * 3600 + 5 * 60 + 6) * 1_000_000_000 + 123_000_000), + ), + ]); + + let batch = RecordBatch::try_new(arrow_schema, vec![Arc::new(intervals)])?; + + let table_schema = TableSchema::new().set_fields([TableFieldSchema::new() + .set_name("duration") + .set_type("INTERVAL")]); + let schema = Arc::new(Schema::new(table_schema)); + + let row0 = Row::try_new_from_arrow(&batch, 0, &schema)?; + let int0: Interval = row0.get("duration"); + assert_eq!( + int0, + Interval { + years: 1, + months: 2, + days: 3, + hours: 4, + minutes: 5, + seconds: 6, + nanos: 789_123_456, + } + ); + + let row1 = Row::try_new_from_arrow(&batch, 1, &schema)?; + let int1: Interval = row1.get("duration"); + assert_eq!( + int1, + Interval { + years: -1, + months: -2, + days: -3, + hours: -4, + minutes: -5, + seconds: -6, + nanos: -123_000_000, + } + ); + + Ok(()) + } } diff --git a/src/bigquery/src/query/schema.rs b/src/bigquery/src/query/schema.rs index 7c3d474328..c71df45d3b 100644 --- a/src/bigquery/src/query/schema.rs +++ b/src/bigquery/src/query/schema.rs @@ -12,8 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use google_cloud_bigquery_v2::model::{TableFieldSchema, TableSchema}; - +use crate::model::{TableFieldSchema, TableSchema}; +use arrow::ipc::reader::StreamReader; +use std::io::Cursor; /// Schema of a table. #[derive(Clone, Debug)] pub(crate) struct Schema(TableSchema); @@ -38,4 +39,118 @@ impl Schema { pub(crate) fn len(&self) -> usize { self.0.fields.len() } + + pub(crate) fn try_from_arrow_ipc( + serialized_schema: &[u8], + ) -> Result { + let reader = StreamReader::try_new(Cursor::new(serialized_schema), None).map_err(|e| { + crate::error::RowError::InvalidRowFormat(format!("failed to parse arrow schema: {e}")) + })?; + let table_schema = table_schema_from_arrow_schema(&reader.schema()); + Ok(Self(table_schema)) + } +} + +fn table_schema_from_arrow_schema(arrow_schema: &arrow::datatypes::Schema) -> TableSchema { + let fields: Vec = arrow_schema + .fields() + .iter() + .map(|f| arrow_field_to_table_field(f.as_ref())) + .collect(); + TableSchema::new().set_fields(fields) +} + +fn arrow_field_to_table_field(field: &arrow::datatypes::Field) -> TableFieldSchema { + use arrow::datatypes::DataType; + + let mode = match field.data_type() { + DataType::List(_) | DataType::LargeList(_) => "REPEATED", + _ if !field.is_nullable() => "REQUIRED", + _ => "NULLABLE", + }; + + let (r#type, nested_fields) = match field.data_type() { + DataType::Null => ("INTEGER", vec![]), + DataType::Boolean => ("BOOLEAN", vec![]), + DataType::Int8 | DataType::Int16 | DataType::Int32 | DataType::Int64 => ("INTEGER", vec![]), + DataType::UInt8 | DataType::UInt16 | DataType::UInt32 | DataType::UInt64 => { + ("INTEGER", vec![]) + } + DataType::Float16 | DataType::Float32 | DataType::Float64 => ("FLOAT", vec![]), + DataType::Utf8 | DataType::LargeUtf8 => ("STRING", vec![]), + DataType::Binary | DataType::LargeBinary => ("BYTES", vec![]), + DataType::Date32 | DataType::Date64 => ("DATE", vec![]), + DataType::Time32(_) | DataType::Time64(_) => ("TIME", vec![]), + DataType::Timestamp(_, Some(_)) => ("TIMESTAMP", vec![]), + DataType::Timestamp(_, None) => ("DATETIME", vec![]), + DataType::Decimal128(_, _) | DataType::Decimal256(_, _) => ("NUMERIC", vec![]), + DataType::Struct(fields) => { + let children = fields + .iter() + .map(|f| arrow_field_to_table_field(f.as_ref())) + .collect(); + ("RECORD", children) + } + DataType::List(inner) | DataType::LargeList(inner) => { + let inner_schema = arrow_field_to_table_field(inner.as_ref()); + return TableFieldSchema::new() + .set_name(field.name().clone()) + .set_type(inner_schema.r#type) + .set_mode("REPEATED") + .set_fields(inner_schema.fields); + } + _ => ("STRING", vec![]), + }; + + TableFieldSchema::new() + .set_name(field.name().clone()) + .set_type(r#type) + .set_mode(mode) + .set_fields(nested_fields) +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::datatypes::{DataType, Field, Schema as ArrowSchema, TimeUnit}; + + #[test] + fn test_from_arrow_schema() { + let arrow_schema = ArrowSchema::new(vec![ + Field::new("name", DataType::Utf8, false), + Field::new("age", DataType::Int64, true), + Field::new( + "tags", + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Utf8, + true, + ))), + true, + ), + Field::new( + "created", + DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())), + false, + ), + ]); + + let table_schema = table_schema_from_arrow_schema(&arrow_schema); + let schema = Schema::new(table_schema); + assert_eq!(schema.len(), 4); + assert_eq!(schema.get_field_index_by_name("name"), Some(0)); + assert_eq!(schema.get_field_index_by_name("age"), Some(1)); + assert_eq!(schema.get_field_index_by_name("tags"), Some(2)); + assert_eq!(schema.get_field_index_by_name("created"), Some(3)); + + let f0 = schema.get_field_by_index(0).unwrap(); + assert_eq!(f0.name, "name"); + assert_eq!(f0.r#type, "STRING"); + assert_eq!(f0.mode, "REQUIRED"); + + let f2 = schema.get_field_by_index(2).unwrap(); + assert_eq!(f2.name, "tags"); + assert_eq!(f2.r#type, "STRING"); + assert_eq!(f2.mode, "REPEATED"); + } } diff --git a/src/generated/cloud/bigquery/v2/src/client.rs b/src/generated/cloud/bigquery/v2/src/client.rs index 9afd429344..4ea94974e3 100644 --- a/src/generated/cloud/bigquery/v2/src/client.rs +++ b/src/generated/cloud/bigquery/v2/src/client.rs @@ -125,6 +125,10 @@ impl DatasetService { /// Returns the dataset specified by datasetID. /// + /// # IAM Permissions + /// + /// Requires the `bigquery.datasets.get` permission on the dataset. + /// /// # Example /// ``` /// # use google_cloud_bigquery_v2::client::DatasetService; @@ -145,6 +149,10 @@ impl DatasetService { /// Creates a new empty dataset. /// + /// # IAM Permissions + /// + /// Requires the `bigquery.datasets.create` permission on the project. + /// /// # Example /// ``` /// # use google_cloud_bigquery_v2::client::DatasetService; @@ -168,6 +176,13 @@ impl DatasetService { /// are provided in the submitted dataset resource. /// This method supports RFC5789 patch semantics. /// + /// # IAM Permissions + /// + /// Requires the following IAM permission(s) to use this method: + /// + /// - `bigquery.datasets.update` on the dataset. + /// - `bigquery.datasets.get` on the dataset. + /// /// # Example /// ``` /// # use google_cloud_bigquery_v2::client::DatasetService; @@ -190,6 +205,10 @@ impl DatasetService { /// entire dataset resource, whereas the patch method only replaces fields that /// are provided in the submitted dataset resource. /// + /// # IAM Permissions + /// + /// Requires the `bigquery.datasets.update` permission on the dataset. + /// /// # Example /// ``` /// # use google_cloud_bigquery_v2::client::DatasetService; @@ -213,6 +232,10 @@ impl DatasetService { /// deleteContents. Immediately after deletion, you can create another dataset /// with the same name. /// + /// # IAM Permissions + /// + /// Requires the `bigquery.datasets.delete` permission on the dataset. + /// /// # Example /// ``` /// # use google_cloud_bigquery_v2::client::DatasetService; @@ -233,6 +256,12 @@ impl DatasetService { /// Lists all datasets in the specified project to which the user has been /// granted the READER dataset role. /// + /// # IAM Permissions + /// + /// Requires no specific IAM permission(s) to use this method. + /// Results are filtered to only include datasets on which the caller has the + /// `bigquery.datasets.get` permission. + /// /// # Example /// ``` /// # use google_cloud_bigquery_v2::client::DatasetService; @@ -258,6 +287,13 @@ impl DatasetService { /// If a time is specified, the dataset version deleted at that time is /// undeleted, else the last live version is undeleted. /// + /// # IAM Permissions + /// + /// Requires the following IAM permission(s) to use this method: + /// + /// - `bigquery.datasets.create` on the project. + /// - `bigquery.datasets.get` on the dataset. + /// /// # Example /// ``` /// # use google_cloud_bigquery_v2::client::DatasetService; @@ -385,6 +421,12 @@ impl JobService { /// the client will need to poll for the job status to see if the cancel /// completed successfully. Cancelled jobs may still incur costs. /// + /// # IAM Permissions + /// + /// Requires the `bigquery.jobs.update` permission on the job resource. + /// If the user matches the creator of the job, the `bigquery.jobs.create` + /// permission on the project is required instead. + /// /// # Example /// ``` /// # use google_cloud_bigquery_v2::client::JobService; @@ -407,6 +449,12 @@ impl JobService { /// a six month period after creation. Requires that you're the person who ran /// the job, or have the Is Owner project role. /// + /// # IAM Permissions + /// + /// Requires the `bigquery.jobs.get` permission on the job resource. + /// If the user matches the creator of the job, the `bigquery.jobs.create` + /// permission on the project is required instead. + /// /// # Example /// ``` /// # use google_cloud_bigquery_v2::client::JobService; @@ -437,6 +485,19 @@ impl JobService { /// accepts the job configuration and the data as two distinct multipart MIME /// parts. /// + /// # IAM Permissions + /// + /// Requires the `bigquery.jobs.create` permission on the project resource. + /// + /// Additional permissions are required depending on the job type: + /// + /// - **Load, Export, and Copy jobs**: Generally require data-level + /// permissions such as `bigquery.tables.export` or access to external + /// storage. + /// - **Query jobs**: Permissions are dependent on the SQL statement. + /// Complex queries (DDL, DCL) may require additional permissions to + /// create reservations, modify IAM policies, or update project settings. + /// /// # Example /// ``` /// # use google_cloud_bigquery_v2::client::JobService; @@ -458,6 +519,10 @@ impl JobService { /// Requests the deletion of the metadata of a job. This call returns when the /// job's metadata is deleted. /// + /// # IAM Permissions + /// + /// Requires the `bigquery.jobs.delete` permission on the job resource. + /// /// # Example /// ``` /// # use google_cloud_bigquery_v2::client::JobService; @@ -481,6 +546,18 @@ impl JobService { /// project role, or the Is Owner project role if you set the allUsers /// property. /// + /// # IAM Permissions + /// + /// Requires no specific IAM permission(s) to use this method. Users are able + /// to list the jobs they created. + /// + /// Additional access is granted based on the following permissions: + /// + /// - Users with the `bigquery.jobs.listAll` permission can list all jobs with + /// all metadata. + /// - Users with the `bigquery.jobs.list` permission can list all jobs, but + /// with redacted information for jobs they did not create. + /// /// # Example /// ``` /// # use google_cloud_bigquery_v2::client::JobService; @@ -504,6 +581,19 @@ impl JobService { /// RPC to get the results of a query job. /// + /// # IAM Permissions + /// + /// Requires the following IAM permission(s) to use this method: + /// + /// - `bigquery.jobs.get` on the job. + /// - `bigquery.tables.getData` on the destination table. + /// + /// If the user matches the creator of the job, the following IAM permission(s) + /// are required instead: + /// + /// - `bigquery.jobs.create` on the project. + /// - `bigquery.tables.getData` on the destination table. + /// /// # Example /// ``` /// # use google_cloud_bigquery_v2::client::JobService; @@ -525,6 +615,15 @@ impl JobService { /// Runs a BigQuery SQL query synchronously and returns query results if the /// query completes within a specified timeout. /// + /// # IAM Permissions + /// + /// Requires the `bigquery.jobs.create` permission on the project resource. + /// + /// Data-level permissions are highly dependent on the SQL statement being + /// executed. While standard queries require data access (such as + /// `bigquery.tables.getData`), complex operations like DDL or DCL may require + /// permissions to manage reservations, IAM policies, or project settings. + /// /// # Example /// ``` /// # use google_cloud_bigquery_v2::client::JobService; @@ -651,6 +750,10 @@ impl ModelService { /// Gets the specified model resource by model ID. /// + /// # IAM Permissions + /// + /// Requires the `bigquery.models.getMetadata` permission on the model. + /// /// # Example /// ``` /// # use google_cloud_bigquery_v2::client::ModelService; @@ -673,6 +776,10 @@ impl ModelService { /// role. After retrieving the list of models, you can get information about a /// particular model by calling the models.get method. /// + /// # IAM Permissions + /// + /// Requires the `bigquery.models.list` permission on the dataset. + /// /// # Example /// ``` /// # use google_cloud_bigquery_v2::client::ModelService; @@ -696,6 +803,10 @@ impl ModelService { /// Patch specific fields in the specified model. /// + /// # IAM Permissions + /// + /// Requires the `bigquery.models.updateMetadata` permission on the model. + /// /// # Example /// ``` /// # use google_cloud_bigquery_v2::client::ModelService; @@ -716,6 +827,10 @@ impl ModelService { /// Deletes the model specified by modelId from the dataset. /// + /// # IAM Permissions + /// + /// Requires the `bigquery.models.delete` permission on the model. + /// /// # Example /// ``` /// # use google_cloud_bigquery_v2::client::ModelService; @@ -840,7 +955,10 @@ impl ProjectService { } /// RPC to get the service account for a project used for interactions with - /// Google Cloud KMS + /// Google Cloud KMS. Requires the `bigquery.jobs.create` permission on the + /// project resource. This permission is required to authorize the retrieval + /// of the project's service identity for technical management tasks like + /// encryption configuration. /// /// # Example /// ``` @@ -968,6 +1086,10 @@ impl RoutineService { /// Gets the specified routine resource by routine ID. /// + /// # IAM Permissions + /// + /// Requires the `bigquery.routines.get` permission on the routine. + /// /// # Example /// ``` /// # use google_cloud_bigquery_v2::client::RoutineService; @@ -988,6 +1110,10 @@ impl RoutineService { /// Creates a new routine in the dataset. /// + /// # IAM Permissions + /// + /// Requires the `bigquery.routines.create` permission on the dataset. + /// /// # Example /// ``` /// # use google_cloud_bigquery_v2::client::RoutineService; @@ -1009,6 +1135,10 @@ impl RoutineService { /// Updates information in an existing routine. The update method replaces the /// entire Routine resource. /// + /// # IAM Permissions + /// + /// Requires the `bigquery.routines.update` permission on the routine. + /// /// # Example /// ``` /// # use google_cloud_bigquery_v2::client::RoutineService; @@ -1029,6 +1159,10 @@ impl RoutineService { /// Deletes the routine specified by routineId from the dataset. /// + /// # IAM Permissions + /// + /// Requires the `bigquery.routines.delete` permission on the routine. + /// /// # Example /// ``` /// # use google_cloud_bigquery_v2::client::RoutineService; @@ -1049,6 +1183,10 @@ impl RoutineService { /// Lists all routines in the specified dataset. Requires the READER dataset /// role. /// + /// # IAM Permissions + /// + /// Requires the `bigquery.routines.list` permission on the dataset. + /// /// # Example /// ``` /// # use google_cloud_bigquery_v2::client::RoutineService; @@ -1179,6 +1317,10 @@ impl RowAccessPolicyService { /// Lists all row access policies on the specified table. /// + /// # IAM Permissions + /// + /// Requires the `bigquery.rowAccessPolicies.list` permission on the table. + /// /// # Example /// ``` /// # use google_cloud_bigquery_v2::client::RowAccessPolicyService; @@ -1204,6 +1346,10 @@ impl RowAccessPolicyService { /// Gets the specified row access policy by policy ID. /// + /// # IAM Permissions + /// + /// Requires the `bigquery.rowAccessPolicies.get` permission on the table. + /// /// # Example /// ``` /// # use google_cloud_bigquery_v2::client::RowAccessPolicyService; @@ -1226,6 +1372,14 @@ impl RowAccessPolicyService { /// Creates a row access policy. /// + /// # IAM Permissions + /// + /// Requires the following IAM permission(s) on the table: + /// + /// - `bigquery.rowAccessPolicies.create` + /// - `bigquery.rowAccessPolicies.setIamPolicy` + /// - `bigquery.tables.getData` + /// /// # Example /// ``` /// # use google_cloud_bigquery_v2::client::RowAccessPolicyService; @@ -1248,6 +1402,14 @@ impl RowAccessPolicyService { /// Updates a row access policy. /// + /// # IAM Permissions + /// + /// Requires the following IAM permission(s) on the table: + /// + /// - `bigquery.rowAccessPolicies.update` + /// - `bigquery.rowAccessPolicies.setIamPolicy` + /// - `bigquery.tables.getData` + /// /// # Example /// ``` /// # use google_cloud_bigquery_v2::client::RowAccessPolicyService; @@ -1270,6 +1432,13 @@ impl RowAccessPolicyService { /// Deletes a row access policy. /// + /// # IAM Permissions + /// + /// Requires the following IAM permission(s) on the table: + /// + /// - `bigquery.rowAccessPolicies.delete` + /// - `bigquery.rowAccessPolicies.setIamPolicy` + /// /// # Example /// ``` /// # use google_cloud_bigquery_v2::client::RowAccessPolicyService; @@ -1291,6 +1460,13 @@ impl RowAccessPolicyService { /// Deletes provided row access policies. /// + /// # IAM Permissions + /// + /// Requires the following IAM permission(s) on the table: + /// + /// - `bigquery.rowAccessPolicies.delete` + /// - `bigquery.rowAccessPolicies.setIamPolicy` + /// /// # Example /// ``` /// # use google_cloud_bigquery_v2::client::RowAccessPolicyService; @@ -1423,6 +1599,10 @@ impl TableService { /// This method does not return the data in the table, it only returns the /// table resource, which describes the structure of this table. /// + /// # IAM Permissions + /// + /// Requires the `bigquery.tables.get` permission on the table. + /// /// # Example /// ``` /// # use google_cloud_bigquery_v2::client::TableService; @@ -1443,6 +1623,10 @@ impl TableService { /// Creates a new, empty table in the dataset. /// + /// # IAM Permissions + /// + /// Requires the `bigquery.tables.create` permission on the dataset. + /// /// # Example /// ``` /// # use google_cloud_bigquery_v2::client::TableService; @@ -1466,6 +1650,13 @@ impl TableService { /// are provided in the submitted table resource. /// This method supports RFC5789 patch semantics. /// + /// # IAM Permissions + /// + /// Requires the following IAM permission(s) on the table: + /// + /// - `bigquery.tables.update` + /// - `bigquery.tables.get` + /// /// # Example /// ``` /// # use google_cloud_bigquery_v2::client::TableService; @@ -1488,6 +1679,10 @@ impl TableService { /// entire Table resource, whereas the patch method only replaces fields that /// are provided in the submitted Table resource. /// + /// # IAM Permissions + /// + /// Requires the `bigquery.tables.update` permission on the table. + /// /// # Example /// ``` /// # use google_cloud_bigquery_v2::client::TableService; @@ -1509,6 +1704,10 @@ impl TableService { /// Deletes the table specified by tableId from the dataset. /// If the table contains data, all the data will be deleted. /// + /// # IAM Permissions + /// + /// Requires the `bigquery.tables.delete` permission on the table. + /// /// # Example /// ``` /// # use google_cloud_bigquery_v2::client::TableService; @@ -1529,6 +1728,10 @@ impl TableService { /// Lists all tables in the specified dataset. Requires the READER dataset /// role. /// + /// # IAM Permissions + /// + /// Requires the `bigquery.tables.list` permission on the dataset. + /// /// # Example /// ``` /// # use google_cloud_bigquery_v2::client::TableService; diff --git a/src/generated/cloud/bigquery/v2/src/model.rs b/src/generated/cloud/bigquery/v2/src/model.rs index 674cd8bc37..0a2a5106f9 100644 --- a/src/generated/cloud/bigquery/v2/src/model.rs +++ b/src/generated/cloud/bigquery/v2/src/model.rs @@ -39,6 +39,454 @@ mod debug; mod deserialize; mod serialize; +/// Arrow schema as specified +/// in and serialized to +/// bytes using IPC: +/// +/// +/// See code samples on how this message can be deserialized. +/// +/// This feature is not yet available. +#[derive(Clone, Default, PartialEq)] +#[non_exhaustive] +pub struct ArrowSchema { + /// IPC serialized Arrow schema. + pub serialized_schema: ::bytes::Bytes, + + pub(crate) _unknown_fields: serde_json::Map, +} + +impl ArrowSchema { + /// Creates a new default instance. + pub fn new() -> Self { + std::default::Default::default() + } + + /// Sets the value of [serialized_schema][crate::model::ArrowSchema::serialized_schema]. + /// + /// # Example + /// ```ignore,no_run + /// # use google_cloud_bigquery_v2::model::ArrowSchema; + /// let x = ArrowSchema::new().set_serialized_schema(bytes::Bytes::from_static(b"example")); + /// ``` + pub fn set_serialized_schema>(mut self, v: T) -> Self { + self.serialized_schema = v.into(); + self + } +} + +impl wkt::message::Message for ArrowSchema { + fn typename() -> &'static str { + "type.googleapis.com/google.cloud.bigquery.v2.ArrowSchema" + } +} + +/// Arrow RecordBatch. +/// +/// This feature is not yet available. +#[derive(Clone, Default, PartialEq)] +#[non_exhaustive] +pub struct ArrowRecordBatch { + /// IPC-serialized Arrow RecordBatch. + pub serialized_record_batch: ::bytes::Bytes, + + pub(crate) _unknown_fields: serde_json::Map, +} + +impl ArrowRecordBatch { + /// Creates a new default instance. + pub fn new() -> Self { + std::default::Default::default() + } + + /// Sets the value of [serialized_record_batch][crate::model::ArrowRecordBatch::serialized_record_batch]. + /// + /// # Example + /// ```ignore,no_run + /// # use google_cloud_bigquery_v2::model::ArrowRecordBatch; + /// let x = ArrowRecordBatch::new().set_serialized_record_batch(bytes::Bytes::from_static(b"example")); + /// ``` + pub fn set_serialized_record_batch>( + mut self, + v: T, + ) -> Self { + self.serialized_record_batch = v.into(); + self + } +} + +impl wkt::message::Message for ArrowRecordBatch { + fn typename() -> &'static str { + "type.googleapis.com/google.cloud.bigquery.v2.ArrowRecordBatch" + } +} + +/// Contains options specific to Arrow Serialization. +/// +/// This feature is not yet available. +#[derive(Clone, Default, PartialEq)] +#[non_exhaustive] +pub struct ArrowSerializationOptions { + /// The compression codec to use for Arrow buffers in serialized record + /// batches. + pub buffer_compression: crate::model::arrow_serialization_options::CompressionCodec, + + /// Optional. Set timestamp precision option. If not set, the default precision + /// is microseconds. + pub picos_timestamp_precision: + crate::model::arrow_serialization_options::PicosTimestampPrecision, + + pub(crate) _unknown_fields: serde_json::Map, +} + +impl ArrowSerializationOptions { + /// Creates a new default instance. + pub fn new() -> Self { + std::default::Default::default() + } + + /// Sets the value of [buffer_compression][crate::model::ArrowSerializationOptions::buffer_compression]. + /// + /// # Example + /// ```ignore,no_run + /// # use google_cloud_bigquery_v2::model::ArrowSerializationOptions; + /// use google_cloud_bigquery_v2::model::arrow_serialization_options::CompressionCodec; + /// let x0 = ArrowSerializationOptions::new().set_buffer_compression(CompressionCodec::Lz4Frame); + /// let x1 = ArrowSerializationOptions::new().set_buffer_compression(CompressionCodec::Zstd); + /// ``` + pub fn set_buffer_compression< + T: std::convert::Into, + >( + mut self, + v: T, + ) -> Self { + self.buffer_compression = v.into(); + self + } + + /// Sets the value of [picos_timestamp_precision][crate::model::ArrowSerializationOptions::picos_timestamp_precision]. + /// + /// # Example + /// ```ignore,no_run + /// # use google_cloud_bigquery_v2::model::ArrowSerializationOptions; + /// use google_cloud_bigquery_v2::model::arrow_serialization_options::PicosTimestampPrecision; + /// let x0 = ArrowSerializationOptions::new().set_picos_timestamp_precision(PicosTimestampPrecision::TimestampPrecisionMicros); + /// let x1 = ArrowSerializationOptions::new().set_picos_timestamp_precision(PicosTimestampPrecision::TimestampPrecisionNanos); + /// let x2 = ArrowSerializationOptions::new().set_picos_timestamp_precision(PicosTimestampPrecision::TimestampPrecisionPicos); + /// ``` + pub fn set_picos_timestamp_precision< + T: std::convert::Into, + >( + mut self, + v: T, + ) -> Self { + self.picos_timestamp_precision = v.into(); + self + } +} + +impl wkt::message::Message for ArrowSerializationOptions { + fn typename() -> &'static str { + "type.googleapis.com/google.cloud.bigquery.v2.ArrowSerializationOptions" + } +} + +/// Defines additional types related to [ArrowSerializationOptions]. +pub mod arrow_serialization_options { + #[allow(unused_imports)] + use super::*; + + /// Compression codec's supported by Arrow. + /// + /// # Working with unknown values + /// + /// This enum is defined as `#[non_exhaustive]` because Google Cloud may add + /// additional enum variants at any time. Adding new variants is not considered + /// a breaking change. Applications should write their code in anticipation of: + /// + /// - New values appearing in future releases of the client library, **and** + /// - New values received dynamically, without application changes. + /// + /// Please consult the [Working with enums] section in the user guide for some + /// guidelines. + /// + /// [Working with enums]: https://googleapis.github.io/google-cloud-rust/working_with_enums.html + #[derive(Clone, Debug, PartialEq)] + #[non_exhaustive] + pub enum CompressionCodec { + /// If unspecified no compression will be used. + CompressionUnspecified, + /// LZ4 Frame () + Lz4Frame, + /// Zstandard compression. + Zstd, + /// If set, the enum was initialized with an unknown value. + /// + /// Applications can examine the value using [CompressionCodec::value] or + /// [CompressionCodec::name]. + UnknownValue(compression_codec::UnknownValue), + } + + #[doc(hidden)] + pub mod compression_codec { + #[allow(unused_imports)] + use super::*; + #[derive(Clone, Debug, PartialEq)] + pub struct UnknownValue(pub(crate) wkt::internal::UnknownEnumValue); + } + + impl CompressionCodec { + /// Gets the enum value. + /// + /// Returns `None` if the enum contains an unknown value deserialized from + /// the string representation of enums. + pub fn value(&self) -> std::option::Option { + match self { + Self::CompressionUnspecified => std::option::Option::Some(0), + Self::Lz4Frame => std::option::Option::Some(1), + Self::Zstd => std::option::Option::Some(2), + Self::UnknownValue(u) => u.0.value(), + } + } + + /// Gets the enum value as a string. + /// + /// Returns `None` if the enum contains an unknown value deserialized from + /// the integer representation of enums. + pub fn name(&self) -> std::option::Option<&str> { + match self { + Self::CompressionUnspecified => { + std::option::Option::Some("COMPRESSION_UNSPECIFIED") + } + Self::Lz4Frame => std::option::Option::Some("LZ4_FRAME"), + Self::Zstd => std::option::Option::Some("ZSTD"), + Self::UnknownValue(u) => u.0.name(), + } + } + } + + impl std::default::Default for CompressionCodec { + fn default() -> Self { + use std::convert::From; + Self::from(0) + } + } + + impl std::fmt::Display for CompressionCodec { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::result::Result<(), std::fmt::Error> { + wkt::internal::display_enum(f, self.name(), self.value()) + } + } + + impl std::convert::From for CompressionCodec { + fn from(value: i32) -> Self { + match value { + 0 => Self::CompressionUnspecified, + 1 => Self::Lz4Frame, + 2 => Self::Zstd, + _ => Self::UnknownValue(compression_codec::UnknownValue( + wkt::internal::UnknownEnumValue::Integer(value), + )), + } + } + } + + impl std::convert::From<&str> for CompressionCodec { + fn from(value: &str) -> Self { + use std::string::ToString; + match value { + "COMPRESSION_UNSPECIFIED" => Self::CompressionUnspecified, + "LZ4_FRAME" => Self::Lz4Frame, + "ZSTD" => Self::Zstd, + _ => Self::UnknownValue(compression_codec::UnknownValue( + wkt::internal::UnknownEnumValue::String(value.to_string()), + )), + } + } + } + + impl serde::ser::Serialize for CompressionCodec { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + match self { + Self::CompressionUnspecified => serializer.serialize_i32(0), + Self::Lz4Frame => serializer.serialize_i32(1), + Self::Zstd => serializer.serialize_i32(2), + Self::UnknownValue(u) => u.0.serialize(serializer), + } + } + } + + impl<'de> serde::de::Deserialize<'de> for CompressionCodec { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_any(wkt::internal::EnumVisitor::::new( + ".google.cloud.bigquery.v2.ArrowSerializationOptions.CompressionCodec", + )) + } + } + + /// The precision of the timestamp value in the Avro message. This precision + /// will **only** be applied to the column(s) with the `TIMESTAMP_PICOS` type. + /// + /// # Working with unknown values + /// + /// This enum is defined as `#[non_exhaustive]` because Google Cloud may add + /// additional enum variants at any time. Adding new variants is not considered + /// a breaking change. Applications should write their code in anticipation of: + /// + /// - New values appearing in future releases of the client library, **and** + /// - New values received dynamically, without application changes. + /// + /// Please consult the [Working with enums] section in the user guide for some + /// guidelines. + /// + /// [Working with enums]: https://googleapis.github.io/google-cloud-rust/working_with_enums.html + #[derive(Clone, Debug, PartialEq)] + #[non_exhaustive] + pub enum PicosTimestampPrecision { + /// Unspecified timestamp precision. The default precision is microseconds. + Unspecified, + /// Timestamp values returned in the results will be truncated to microsecond + /// level precision. The value will be encoded as Arrow TIMESTAMP type in a + /// 64 bit integer. + TimestampPrecisionMicros, + /// Timestamp values returned in the results will be truncated to nanosecond + /// level precision. The value will be encoded as Arrow TIMESTAMP type in a + /// 64 bit integer. + TimestampPrecisionNanos, + /// Timestamp values returned in the results will contain full precision + /// picosecond value. The value will be encoded as a string which conforms to + /// ISO 8601 format. + TimestampPrecisionPicos, + /// If set, the enum was initialized with an unknown value. + /// + /// Applications can examine the value using [PicosTimestampPrecision::value] or + /// [PicosTimestampPrecision::name]. + UnknownValue(picos_timestamp_precision::UnknownValue), + } + + #[doc(hidden)] + pub mod picos_timestamp_precision { + #[allow(unused_imports)] + use super::*; + #[derive(Clone, Debug, PartialEq)] + pub struct UnknownValue(pub(crate) wkt::internal::UnknownEnumValue); + } + + impl PicosTimestampPrecision { + /// Gets the enum value. + /// + /// Returns `None` if the enum contains an unknown value deserialized from + /// the string representation of enums. + pub fn value(&self) -> std::option::Option { + match self { + Self::Unspecified => std::option::Option::Some(0), + Self::TimestampPrecisionMicros => std::option::Option::Some(1), + Self::TimestampPrecisionNanos => std::option::Option::Some(2), + Self::TimestampPrecisionPicos => std::option::Option::Some(3), + Self::UnknownValue(u) => u.0.value(), + } + } + + /// Gets the enum value as a string. + /// + /// Returns `None` if the enum contains an unknown value deserialized from + /// the integer representation of enums. + pub fn name(&self) -> std::option::Option<&str> { + match self { + Self::Unspecified => { + std::option::Option::Some("PICOS_TIMESTAMP_PRECISION_UNSPECIFIED") + } + Self::TimestampPrecisionMicros => { + std::option::Option::Some("TIMESTAMP_PRECISION_MICROS") + } + Self::TimestampPrecisionNanos => { + std::option::Option::Some("TIMESTAMP_PRECISION_NANOS") + } + Self::TimestampPrecisionPicos => { + std::option::Option::Some("TIMESTAMP_PRECISION_PICOS") + } + Self::UnknownValue(u) => u.0.name(), + } + } + } + + impl std::default::Default for PicosTimestampPrecision { + fn default() -> Self { + use std::convert::From; + Self::from(0) + } + } + + impl std::fmt::Display for PicosTimestampPrecision { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::result::Result<(), std::fmt::Error> { + wkt::internal::display_enum(f, self.name(), self.value()) + } + } + + impl std::convert::From for PicosTimestampPrecision { + fn from(value: i32) -> Self { + match value { + 0 => Self::Unspecified, + 1 => Self::TimestampPrecisionMicros, + 2 => Self::TimestampPrecisionNanos, + 3 => Self::TimestampPrecisionPicos, + _ => Self::UnknownValue(picos_timestamp_precision::UnknownValue( + wkt::internal::UnknownEnumValue::Integer(value), + )), + } + } + } + + impl std::convert::From<&str> for PicosTimestampPrecision { + fn from(value: &str) -> Self { + use std::string::ToString; + match value { + "PICOS_TIMESTAMP_PRECISION_UNSPECIFIED" => Self::Unspecified, + "TIMESTAMP_PRECISION_MICROS" => Self::TimestampPrecisionMicros, + "TIMESTAMP_PRECISION_NANOS" => Self::TimestampPrecisionNanos, + "TIMESTAMP_PRECISION_PICOS" => Self::TimestampPrecisionPicos, + _ => Self::UnknownValue(picos_timestamp_precision::UnknownValue( + wkt::internal::UnknownEnumValue::String(value.to_string()), + )), + } + } + } + + impl serde::ser::Serialize for PicosTimestampPrecision { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + match self { + Self::Unspecified => serializer.serialize_i32(0), + Self::TimestampPrecisionMicros => serializer.serialize_i32(1), + Self::TimestampPrecisionNanos => serializer.serialize_i32(2), + Self::TimestampPrecisionPicos => serializer.serialize_i32(3), + Self::UnknownValue(u) => u.0.serialize(serializer), + } + } + } + + impl<'de> serde::de::Deserialize<'de> for PicosTimestampPrecision { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_any( + wkt::internal::EnumVisitor::::new( + ".google.cloud.bigquery.v2.ArrowSerializationOptions.PicosTimestampPrecision", + ), + ) + } + } +} + /// Configuration for BigQuery tables for Apache Iceberg (formerly BigLake /// managed tables.) #[derive(Clone, Default, PartialEq)] @@ -7585,8 +8033,9 @@ pub struct GenAiStats { /// Job level error stats across all GenAi functions pub error_stats: std::option::Option, - /// Function level stats for GenAi Functions. - /// See + /// Function level stats for GenAI Functions. For more information, see + /// [Generative AI + /// overview](https://docs.cloud.google.com/bigquery/docs/generative-ai-overview). pub function_stats: std::vec::Vec, pub(crate) _unknown_fields: serde_json::Map, @@ -10237,6 +10686,9 @@ pub struct QueryRequest { /// Optional. The reservation that jobs.query request would use. User can /// specify a reservation to execute the job.query. The expected format is /// `projects/{project}/locations/{location}/reservations/{reservation}`. + /// Forces the query to use on-demand billing when set to `none`. + /// This requires the project or organization to have + /// `reservation_override_mode` set to `ALLOW_ANY_OVERRIDE`. pub reservation: std::option::Option, /// Optional. This is only supported for SELECT query. If set, the query is @@ -10245,6 +10697,27 @@ pub struct QueryRequest { /// SQL. This feature is not yet available. pub write_incremental_results: bool, + /// Optional. The query results format. + /// If the value is anything other than `STRUCT_ENCODING` or unspecified: + /// + /// * The schema of the results will be provided in + /// `QueryResponse.results_schema` field. + /// * The results of the first page will be provided in + /// `QueryResponse.results` field. + /// * The `QueryResponse.rows` will not be populated. + /// * The `QueryResponse.schema` for `QueryResponse.rows` will also not be + /// populated since it is the schema of the `QueryResponse.rows`. + /// + /// This feature is not yet available. + pub query_results_format: crate::model::query_request::QueryResultsFormat, + + /// Results serialization options for `query_results_format` other than + /// `STRUCT_ENCODING`. + /// + /// This feature is not yet available. + pub results_format_serialization_options: + std::option::Option, + pub(crate) _unknown_fields: serde_json::Map, } @@ -10814,6 +11287,89 @@ impl QueryRequest { self.write_incremental_results = v.into(); self } + + /// Sets the value of [query_results_format][crate::model::QueryRequest::query_results_format]. + /// + /// # Example + /// ```ignore,no_run + /// # use google_cloud_bigquery_v2::model::QueryRequest; + /// use google_cloud_bigquery_v2::model::query_request::QueryResultsFormat; + /// let x0 = QueryRequest::new().set_query_results_format(QueryResultsFormat::StructEncoding); + /// let x1 = QueryRequest::new().set_query_results_format(QueryResultsFormat::Arrow); + /// ``` + pub fn set_query_results_format< + T: std::convert::Into, + >( + mut self, + v: T, + ) -> Self { + self.query_results_format = v.into(); + self + } + + /// Sets the value of [results_format_serialization_options][crate::model::QueryRequest::results_format_serialization_options]. + /// + /// Note that all the setters affecting `results_format_serialization_options` are mutually + /// exclusive. + /// + /// # Example + /// ```ignore,no_run + /// # use google_cloud_bigquery_v2::model::QueryRequest; + /// use google_cloud_bigquery_v2::model::ArrowSerializationOptions; + /// let x = QueryRequest::new().set_results_format_serialization_options(Some( + /// google_cloud_bigquery_v2::model::query_request::ResultsFormatSerializationOptions::ArrowSerializationOptions(ArrowSerializationOptions::default().into()))); + /// ``` + pub fn set_results_format_serialization_options< + T: std::convert::Into< + std::option::Option, + >, + >( + mut self, + v: T, + ) -> Self { + self.results_format_serialization_options = v.into(); + self + } + + /// The value of [results_format_serialization_options][crate::model::QueryRequest::results_format_serialization_options] + /// if it holds a `ArrowSerializationOptions`, `None` if the field is not set or + /// holds a different branch. + pub fn arrow_serialization_options( + &self, + ) -> std::option::Option<&std::boxed::Box> { + #[allow(unreachable_patterns)] + self.results_format_serialization_options.as_ref().and_then(|v| match v { + crate::model::query_request::ResultsFormatSerializationOptions::ArrowSerializationOptions(v) => std::option::Option::Some(v), + _ => std::option::Option::None, + }) + } + + /// Sets the value of [results_format_serialization_options][crate::model::QueryRequest::results_format_serialization_options] + /// to hold a `ArrowSerializationOptions`. + /// + /// Note that all the setters affecting `results_format_serialization_options` are + /// mutually exclusive. + /// + /// # Example + /// ```ignore,no_run + /// # use google_cloud_bigquery_v2::model::QueryRequest; + /// use google_cloud_bigquery_v2::model::ArrowSerializationOptions; + /// let x = QueryRequest::new().set_arrow_serialization_options(ArrowSerializationOptions::default()/* use setters */); + /// assert!(x.arrow_serialization_options().is_some()); + /// ``` + pub fn set_arrow_serialization_options< + T: std::convert::Into>, + >( + mut self, + v: T, + ) -> Self { + self.results_format_serialization_options = std::option::Option::Some( + crate::model::query_request::ResultsFormatSerializationOptions::ArrowSerializationOptions( + v.into() + ) + ); + self + } } impl wkt::message::Message for QueryRequest { @@ -10962,6 +11518,153 @@ pub mod query_request { )) } } + + /// The format of the query results. + /// + /// This feature is not yet available. + /// + /// # Working with unknown values + /// + /// This enum is defined as `#[non_exhaustive]` because Google Cloud may add + /// additional enum variants at any time. Adding new variants is not considered + /// a breaking change. Applications should write their code in anticipation of: + /// + /// - New values appearing in future releases of the client library, **and** + /// - New values received dynamically, without application changes. + /// + /// Please consult the [Working with enums] section in the user guide for some + /// guidelines. + /// + /// [Working with enums]: https://googleapis.github.io/google-cloud-rust/working_with_enums.html + #[derive(Clone, Debug, PartialEq)] + #[non_exhaustive] + pub enum QueryResultsFormat { + /// If unspecified it will default to struct `QueryResponse.rows` + /// (`STRUCT_ENCODING`) + Unspecified, + /// Default encoding of results as struct in `QueryResponse.rows` + StructEncoding, + /// Arrow is a standard open source column-based message format. + /// See for more details. + Arrow, + /// If set, the enum was initialized with an unknown value. + /// + /// Applications can examine the value using [QueryResultsFormat::value] or + /// [QueryResultsFormat::name]. + UnknownValue(query_results_format::UnknownValue), + } + + #[doc(hidden)] + pub mod query_results_format { + #[allow(unused_imports)] + use super::*; + #[derive(Clone, Debug, PartialEq)] + pub struct UnknownValue(pub(crate) wkt::internal::UnknownEnumValue); + } + + impl QueryResultsFormat { + /// Gets the enum value. + /// + /// Returns `None` if the enum contains an unknown value deserialized from + /// the string representation of enums. + pub fn value(&self) -> std::option::Option { + match self { + Self::Unspecified => std::option::Option::Some(0), + Self::StructEncoding => std::option::Option::Some(1), + Self::Arrow => std::option::Option::Some(2), + Self::UnknownValue(u) => u.0.value(), + } + } + + /// Gets the enum value as a string. + /// + /// Returns `None` if the enum contains an unknown value deserialized from + /// the integer representation of enums. + pub fn name(&self) -> std::option::Option<&str> { + match self { + Self::Unspecified => std::option::Option::Some("QUERY_RESULTS_FORMAT_UNSPECIFIED"), + Self::StructEncoding => std::option::Option::Some("STRUCT_ENCODING"), + Self::Arrow => std::option::Option::Some("ARROW"), + Self::UnknownValue(u) => u.0.name(), + } + } + } + + impl std::default::Default for QueryResultsFormat { + fn default() -> Self { + use std::convert::From; + Self::from(0) + } + } + + impl std::fmt::Display for QueryResultsFormat { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::result::Result<(), std::fmt::Error> { + wkt::internal::display_enum(f, self.name(), self.value()) + } + } + + impl std::convert::From for QueryResultsFormat { + fn from(value: i32) -> Self { + match value { + 0 => Self::Unspecified, + 1 => Self::StructEncoding, + 2 => Self::Arrow, + _ => Self::UnknownValue(query_results_format::UnknownValue( + wkt::internal::UnknownEnumValue::Integer(value), + )), + } + } + } + + impl std::convert::From<&str> for QueryResultsFormat { + fn from(value: &str) -> Self { + use std::string::ToString; + match value { + "QUERY_RESULTS_FORMAT_UNSPECIFIED" => Self::Unspecified, + "STRUCT_ENCODING" => Self::StructEncoding, + "ARROW" => Self::Arrow, + _ => Self::UnknownValue(query_results_format::UnknownValue( + wkt::internal::UnknownEnumValue::String(value.to_string()), + )), + } + } + } + + impl serde::ser::Serialize for QueryResultsFormat { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + match self { + Self::Unspecified => serializer.serialize_i32(0), + Self::StructEncoding => serializer.serialize_i32(1), + Self::Arrow => serializer.serialize_i32(2), + Self::UnknownValue(u) => u.0.serialize(serializer), + } + } + } + + impl<'de> serde::de::Deserialize<'de> for QueryResultsFormat { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_any(wkt::internal::EnumVisitor::::new( + ".google.cloud.bigquery.v2.QueryRequest.QueryResultsFormat", + )) + } + } + + /// Results serialization options for `query_results_format` other than + /// `STRUCT_ENCODING`. + /// + /// This feature is not yet available. + #[derive(Clone, Debug, PartialEq)] + #[non_exhaustive] + pub enum ResultsFormatSerializationOptions { + /// Optional. Options specific to the Apache Arrow output format. + ArrowSerializationOptions(std::boxed::Box), + } } #[allow(missing_docs)] @@ -11071,6 +11774,151 @@ pub struct QueryResponse { /// field will be present whenever a query job is in the DONE state. pub end_time: std::option::Option, + /// Output only. The number of rows out of `total_rows` returned in this + /// response. + /// + /// This feature is not yet available. + pub page_row_count: i64, + + /// Output only. The type of query statement, if valid. + /// Possible values: + /// + /// * `SELECT`: + /// [`SELECT`](https://cloud.google.com/bigquery/docs/reference/standard-sql/query-syntax#select_list) + /// statement. + /// * `ASSERT`: + /// [`ASSERT`](https://cloud.google.com/bigquery/docs/reference/standard-sql/debugging-statements#assert) + /// statement. + /// * `INSERT`: + /// [`INSERT`](https://cloud.google.com/bigquery/docs/reference/standard-sql/dml-syntax#insert_statement) + /// statement. + /// * `UPDATE`: + /// [`UPDATE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/dml-syntax#update_statement) + /// statement. + /// * `DELETE`: + /// [`DELETE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-manipulation-language) + /// statement. + /// * `MERGE`: + /// [`MERGE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-manipulation-language) + /// statement. + /// * `CREATE_TABLE`: [`CREATE + /// TABLE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_table_statement) + /// statement, without `AS SELECT`. + /// * `CREATE_TABLE_AS_SELECT`: [`CREATE TABLE AS + /// SELECT`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_table_statement) + /// statement. + /// * `CREATE_VIEW`: [`CREATE + /// VIEW`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_view_statement) + /// statement. + /// * `CREATE_MODEL`: [`CREATE + /// MODEL`](https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-create#create_model_statement) + /// statement. + /// * `CREATE_MATERIALIZED_VIEW`: [`CREATE MATERIALIZED + /// VIEW`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_materialized_view_statement) + /// statement. + /// * `CREATE_FUNCTION`: [`CREATE + /// FUNCTION`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_function_statement) + /// statement. + /// * `CREATE_TABLE_FUNCTION`: [`CREATE TABLE + /// FUNCTION`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_table_function_statement) + /// statement. + /// * `CREATE_PROCEDURE`: [`CREATE + /// PROCEDURE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_procedure) + /// statement. + /// * `CREATE_ROW_ACCESS_POLICY`: [`CREATE ROW ACCESS + /// POLICY`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_row_access_policy_statement) + /// statement. + /// * `CREATE_SCHEMA`: [`CREATE + /// SCHEMA`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_schema_statement) + /// statement. + /// * `CREATE_SNAPSHOT_TABLE`: [`CREATE SNAPSHOT + /// TABLE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_snapshot_table_statement) + /// statement. + /// * `CREATE_SEARCH_INDEX`: [`CREATE SEARCH + /// INDEX`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_search_index_statement) + /// statement. + /// * `DROP_TABLE`: [`DROP + /// TABLE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#drop_table_statement) + /// statement. + /// * `DROP_EXTERNAL_TABLE`: [`DROP EXTERNAL + /// TABLE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#drop_external_table_statement) + /// statement. + /// * `DROP_VIEW`: [`DROP + /// VIEW`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#drop_view_statement) + /// statement. + /// * `DROP_MODEL`: [`DROP + /// MODEL`](https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-drop-model) + /// statement. + /// * `DROP_MATERIALIZED_VIEW`: [`DROP MATERIALIZED + /// VIEW`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#drop_materialized_view_statement) + /// statement. + /// * `DROP_FUNCTION` : [`DROP + /// FUNCTION`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#drop_function_statement) + /// statement. + /// * `DROP_TABLE_FUNCTION` : [`DROP TABLE + /// FUNCTION`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#drop_table_function) + /// statement. + /// * `DROP_PROCEDURE`: [`DROP + /// PROCEDURE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#drop_procedure_statement) + /// statement. + /// * `DROP_SEARCH_INDEX`: [`DROP SEARCH + /// INDEX`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#drop_search_index) + /// statement. + /// * `DROP_SCHEMA`: [`DROP + /// SCHEMA`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#drop_schema_statement) + /// statement. + /// * `DROP_SNAPSHOT_TABLE`: [`DROP SNAPSHOT + /// TABLE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#drop_snapshot_table_statement) + /// statement. + /// * `DROP_ROW_ACCESS_POLICY`: [`DROP [ALL] ROW ACCESS + /// POLICY|POLICIES`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#drop_row_access_policy_statement) + /// statement. + /// * `ALTER_TABLE`: [`ALTER + /// TABLE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#alter_table_set_options_statement) + /// statement. + /// * `ALTER_VIEW`: [`ALTER + /// VIEW`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#alter_view_set_options_statement) + /// statement. + /// * `ALTER_MATERIALIZED_VIEW`: [`ALTER MATERIALIZED + /// VIEW`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#alter_materialized_view_set_options_statement) + /// statement. + /// * `ALTER_SCHEMA`: [`ALTER + /// SCHEMA`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#alter_schema_set_options_statement) + /// statement. + /// * `SCRIPT`: + /// [`SCRIPT`](https://cloud.google.com/bigquery/docs/reference/standard-sql/procedural-language). + /// * `TRUNCATE_TABLE`: [`TRUNCATE + /// TABLE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/dml-syntax#truncate_table_statement) + /// statement. + /// * `CREATE_EXTERNAL_TABLE`: [`CREATE EXTERNAL + /// TABLE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_external_table_statement) + /// statement. + /// * `EXPORT_DATA`: [`EXPORT + /// DATA`](https://cloud.google.com/bigquery/docs/reference/standard-sql/other-statements#export_data_statement) + /// statement. + /// * `EXPORT_MODEL`: [`EXPORT + /// MODEL`](https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-export-model) + /// statement. + /// * `LOAD_DATA`: [`LOAD + /// DATA`](https://cloud.google.com/bigquery/docs/reference/standard-sql/other-statements#load_data_statement) + /// statement. + /// * `CALL`: + /// [`CALL`](https://cloud.google.com/bigquery/docs/reference/standard-sql/procedural-language#call) + /// statement. + pub statement_type: std::string::String, + + /// The schema if `query_results_format` has been specified with value other + /// than default `STRUCT_ENCODING`. + /// + /// This feature is not yet available. + pub results_schema: std::option::Option, + + /// Row data is returned in format when `query_results_format` has been + /// specified with value other than default `STRUCT_ENCODING`. + /// + /// This feature is not yet available. + pub results: std::option::Option, + pub(crate) _unknown_fields: serde_json::Map, } @@ -11656,6 +12504,150 @@ impl QueryResponse { self.end_time = v.map(|x| x.into()); self } + + /// Sets the value of [page_row_count][crate::model::QueryResponse::page_row_count]. + /// + /// # Example + /// ```ignore,no_run + /// # use google_cloud_bigquery_v2::model::QueryResponse; + /// let x = QueryResponse::new().set_page_row_count(42); + /// ``` + pub fn set_page_row_count>(mut self, v: T) -> Self { + self.page_row_count = v.into(); + self + } + + /// Sets the value of [statement_type][crate::model::QueryResponse::statement_type]. + /// + /// # Example + /// ```ignore,no_run + /// # use google_cloud_bigquery_v2::model::QueryResponse; + /// let x = QueryResponse::new().set_statement_type("example"); + /// ``` + pub fn set_statement_type>(mut self, v: T) -> Self { + self.statement_type = v.into(); + self + } + + /// Sets the value of [results_schema][crate::model::QueryResponse::results_schema]. + /// + /// Note that all the setters affecting `results_schema` are mutually + /// exclusive. + /// + /// # Example + /// ```ignore,no_run + /// # use google_cloud_bigquery_v2::model::QueryResponse; + /// use google_cloud_bigquery_v2::model::ArrowSchema; + /// let x = QueryResponse::new().set_results_schema(Some( + /// google_cloud_bigquery_v2::model::query_response::ResultsSchema::ArrowSchema(ArrowSchema::default().into()))); + /// ``` + pub fn set_results_schema< + T: std::convert::Into>, + >( + mut self, + v: T, + ) -> Self { + self.results_schema = v.into(); + self + } + + /// The value of [results_schema][crate::model::QueryResponse::results_schema] + /// if it holds a `ArrowSchema`, `None` if the field is not set or + /// holds a different branch. + pub fn arrow_schema(&self) -> std::option::Option<&std::boxed::Box> { + #[allow(unreachable_patterns)] + self.results_schema.as_ref().and_then(|v| match v { + crate::model::query_response::ResultsSchema::ArrowSchema(v) => { + std::option::Option::Some(v) + } + _ => std::option::Option::None, + }) + } + + /// Sets the value of [results_schema][crate::model::QueryResponse::results_schema] + /// to hold a `ArrowSchema`. + /// + /// Note that all the setters affecting `results_schema` are + /// mutually exclusive. + /// + /// # Example + /// ```ignore,no_run + /// # use google_cloud_bigquery_v2::model::QueryResponse; + /// use google_cloud_bigquery_v2::model::ArrowSchema; + /// let x = QueryResponse::new().set_arrow_schema(ArrowSchema::default()/* use setters */); + /// assert!(x.arrow_schema().is_some()); + /// ``` + pub fn set_arrow_schema>>( + mut self, + v: T, + ) -> Self { + self.results_schema = std::option::Option::Some( + crate::model::query_response::ResultsSchema::ArrowSchema(v.into()), + ); + self + } + + /// Sets the value of [results][crate::model::QueryResponse::results]. + /// + /// Note that all the setters affecting `results` are mutually + /// exclusive. + /// + /// # Example + /// ```ignore,no_run + /// # use google_cloud_bigquery_v2::model::QueryResponse; + /// use google_cloud_bigquery_v2::model::ArrowRecordBatch; + /// let x = QueryResponse::new().set_results(Some( + /// google_cloud_bigquery_v2::model::query_response::Results::ArrowRecordBatch(ArrowRecordBatch::default().into()))); + /// ``` + pub fn set_results< + T: std::convert::Into>, + >( + mut self, + v: T, + ) -> Self { + self.results = v.into(); + self + } + + /// The value of [results][crate::model::QueryResponse::results] + /// if it holds a `ArrowRecordBatch`, `None` if the field is not set or + /// holds a different branch. + pub fn arrow_record_batch( + &self, + ) -> std::option::Option<&std::boxed::Box> { + #[allow(unreachable_patterns)] + self.results.as_ref().and_then(|v| match v { + crate::model::query_response::Results::ArrowRecordBatch(v) => { + std::option::Option::Some(v) + } + _ => std::option::Option::None, + }) + } + + /// Sets the value of [results][crate::model::QueryResponse::results] + /// to hold a `ArrowRecordBatch`. + /// + /// Note that all the setters affecting `results` are + /// mutually exclusive. + /// + /// # Example + /// ```ignore,no_run + /// # use google_cloud_bigquery_v2::model::QueryResponse; + /// use google_cloud_bigquery_v2::model::ArrowRecordBatch; + /// let x = QueryResponse::new().set_arrow_record_batch(ArrowRecordBatch::default()/* use setters */); + /// assert!(x.arrow_record_batch().is_some()); + /// ``` + pub fn set_arrow_record_batch< + T: std::convert::Into>, + >( + mut self, + v: T, + ) -> Self { + self.results = std::option::Option::Some( + crate::model::query_response::Results::ArrowRecordBatch(v.into()), + ); + self + } } impl wkt::message::Message for QueryResponse { @@ -11664,6 +12656,34 @@ impl wkt::message::Message for QueryResponse { } } +/// Defines additional types related to [QueryResponse]. +pub mod query_response { + #[allow(unused_imports)] + use super::*; + + /// The schema if `query_results_format` has been specified with value other + /// than default `STRUCT_ENCODING`. + /// + /// This feature is not yet available. + #[derive(Clone, Debug, PartialEq)] + #[non_exhaustive] + pub enum ResultsSchema { + /// Output only. Arrow schema + ArrowSchema(std::boxed::Box), + } + + /// Row data is returned in format when `query_results_format` has been + /// specified with value other than default `STRUCT_ENCODING`. + /// + /// This feature is not yet available. + #[derive(Clone, Debug, PartialEq)] + #[non_exhaustive] + pub enum Results { + /// Output only. Serialized row data in Arrow RecordBatch format. + ArrowRecordBatch(std::boxed::Box), + } +} + /// Properties for the destination table. #[derive(Clone, Default, PartialEq)] #[non_exhaustive] @@ -15702,6 +16722,9 @@ pub struct JobConfiguration { /// determined based on the rules defined by the reservation assignments. The /// expected format is /// `projects/{project}/locations/{location}/reservations/{reservation}`. + /// Forces the query to use on-demand billing when set to `none`, which + /// requires the project or organization to have + /// `reservation_override_mode` set to `ALLOW_ANY_OVERRIDE`. pub reservation: std::option::Option, pub(crate) _unknown_fields: serde_json::Map, @@ -19870,7 +20893,10 @@ pub mod search_statistics { #[allow(unused_imports)] use super::*; - /// Indicates the type of search index usage in the entire search query. + /// Indicates the type of search index usage in the entire search query. In + /// this context, "usage" means that an index lookup is attempted to prune + /// base table data, with effectiveness depending on the selectivity of the + /// search term. /// /// # Working with unknown values /// @@ -21189,8 +22215,11 @@ pub struct JobStatistics2 { /// Output only. Statistics of a Spark procedure job. pub spark_statistics: std::option::Option, - /// Output only. Total bytes transferred for cross-cloud queries such as Cross - /// Cloud Transfer and CREATE TABLE AS SELECT (CTAS). + /// Output only. Total bytes transferred for BigQuery Omni queries from the + /// remote cloud back to Google Cloud. This tracks data movement over + /// Google-managed connections (like query results). It doesn't include input + /// data read from the external data lake (for example, S3) because that data + /// stays within the remote cloud. pub transferred_bytes: std::option::Option, /// Output only. Statistics of materialized views of a query job. @@ -21207,6 +22236,10 @@ pub struct JobStatistics2 { /// Output only. Statistics related to GenAI usage in the query. pub gen_ai_stats: std::option::Option, + /// Output only. Storage and caching statistics per cloud provider for queries + /// over object storage. + pub object_storage_stats: std::vec::Vec, + pub(crate) _unknown_fields: serde_json::Map, } @@ -22534,6 +23567,28 @@ impl JobStatistics2 { self.gen_ai_stats = v.map(|x| x.into()); self } + + /// Sets the value of [object_storage_stats][crate::model::JobStatistics2::object_storage_stats]. + /// + /// # Example + /// ```ignore,no_run + /// # use google_cloud_bigquery_v2::model::JobStatistics2; + /// use google_cloud_bigquery_v2::model::ObjectStorageStats; + /// let x = JobStatistics2::new() + /// .set_object_storage_stats([ + /// ObjectStorageStats::default()/* use setters */, + /// ObjectStorageStats::default()/* use (different) setters */, + /// ]); + /// ``` + pub fn set_object_storage_stats(mut self, v: T) -> Self + where + T: std::iter::IntoIterator, + V: std::convert::Into, + { + use std::iter::Iterator; + self.object_storage_stats = v.into_iter().map(|i| i.into()).collect(); + self + } } impl wkt::message::Message for JobStatistics2 { @@ -22542,6 +23597,280 @@ impl wkt::message::Message for JobStatistics2 { } } +/// Storage and caching statistics for object storage. +#[derive(Clone, Default, PartialEq)] +#[non_exhaustive] +pub struct ObjectStorageStats { + /// The cloud provider for this block of statistics. + pub cloud_provider: std::option::Option, + + /// Total bytes read directly from the cloud provider's storage. + pub object_storage_bytes_read: std::option::Option, + + /// Total bytes read from the GCP Lakehouse-internal cache, avoiding an + /// object storage read. + pub cache_bytes_read: std::option::Option, + + pub(crate) _unknown_fields: serde_json::Map, +} + +impl ObjectStorageStats { + /// Creates a new default instance. + pub fn new() -> Self { + std::default::Default::default() + } + + /// Sets the value of [cloud_provider][crate::model::ObjectStorageStats::cloud_provider]. + /// + /// # Example + /// ```ignore,no_run + /// # use google_cloud_bigquery_v2::model::ObjectStorageStats; + /// use google_cloud_bigquery_v2::model::object_storage_stats::CloudProvider; + /// let x0 = ObjectStorageStats::new().set_cloud_provider(CloudProvider::Gcp); + /// let x1 = ObjectStorageStats::new().set_cloud_provider(CloudProvider::Aws); + /// let x2 = ObjectStorageStats::new().set_cloud_provider(CloudProvider::Azure); + /// ``` + pub fn set_cloud_provider(mut self, v: T) -> Self + where + T: std::convert::Into, + { + self.cloud_provider = std::option::Option::Some(v.into()); + self + } + + /// Sets or clears the value of [cloud_provider][crate::model::ObjectStorageStats::cloud_provider]. + /// + /// # Example + /// ```ignore,no_run + /// # use google_cloud_bigquery_v2::model::ObjectStorageStats; + /// use google_cloud_bigquery_v2::model::object_storage_stats::CloudProvider; + /// let x0 = ObjectStorageStats::new().set_or_clear_cloud_provider(Some(CloudProvider::Gcp)); + /// let x1 = ObjectStorageStats::new().set_or_clear_cloud_provider(Some(CloudProvider::Aws)); + /// let x2 = ObjectStorageStats::new().set_or_clear_cloud_provider(Some(CloudProvider::Azure)); + /// let x_none = ObjectStorageStats::new().set_or_clear_cloud_provider(None::); + /// ``` + pub fn set_or_clear_cloud_provider(mut self, v: std::option::Option) -> Self + where + T: std::convert::Into, + { + self.cloud_provider = v.map(|x| x.into()); + self + } + + /// Sets the value of [object_storage_bytes_read][crate::model::ObjectStorageStats::object_storage_bytes_read]. + /// + /// # Example + /// ```ignore,no_run + /// # use google_cloud_bigquery_v2::model::ObjectStorageStats; + /// let x = ObjectStorageStats::new().set_object_storage_bytes_read(42); + /// ``` + pub fn set_object_storage_bytes_read(mut self, v: T) -> Self + where + T: std::convert::Into, + { + self.object_storage_bytes_read = std::option::Option::Some(v.into()); + self + } + + /// Sets or clears the value of [object_storage_bytes_read][crate::model::ObjectStorageStats::object_storage_bytes_read]. + /// + /// # Example + /// ```ignore,no_run + /// # use google_cloud_bigquery_v2::model::ObjectStorageStats; + /// let x = ObjectStorageStats::new().set_or_clear_object_storage_bytes_read(Some(42)); + /// let x = ObjectStorageStats::new().set_or_clear_object_storage_bytes_read(None::); + /// ``` + pub fn set_or_clear_object_storage_bytes_read(mut self, v: std::option::Option) -> Self + where + T: std::convert::Into, + { + self.object_storage_bytes_read = v.map(|x| x.into()); + self + } + + /// Sets the value of [cache_bytes_read][crate::model::ObjectStorageStats::cache_bytes_read]. + /// + /// # Example + /// ```ignore,no_run + /// # use google_cloud_bigquery_v2::model::ObjectStorageStats; + /// let x = ObjectStorageStats::new().set_cache_bytes_read(42); + /// ``` + pub fn set_cache_bytes_read(mut self, v: T) -> Self + where + T: std::convert::Into, + { + self.cache_bytes_read = std::option::Option::Some(v.into()); + self + } + + /// Sets or clears the value of [cache_bytes_read][crate::model::ObjectStorageStats::cache_bytes_read]. + /// + /// # Example + /// ```ignore,no_run + /// # use google_cloud_bigquery_v2::model::ObjectStorageStats; + /// let x = ObjectStorageStats::new().set_or_clear_cache_bytes_read(Some(42)); + /// let x = ObjectStorageStats::new().set_or_clear_cache_bytes_read(None::); + /// ``` + pub fn set_or_clear_cache_bytes_read(mut self, v: std::option::Option) -> Self + where + T: std::convert::Into, + { + self.cache_bytes_read = v.map(|x| x.into()); + self + } +} + +impl wkt::message::Message for ObjectStorageStats { + fn typename() -> &'static str { + "type.googleapis.com/google.cloud.bigquery.v2.ObjectStorageStats" + } +} + +/// Defines additional types related to [ObjectStorageStats]. +pub mod object_storage_stats { + #[allow(unused_imports)] + use super::*; + + /// The cloud provider hosting the object storage. + /// + /// # Working with unknown values + /// + /// This enum is defined as `#[non_exhaustive]` because Google Cloud may add + /// additional enum variants at any time. Adding new variants is not considered + /// a breaking change. Applications should write their code in anticipation of: + /// + /// - New values appearing in future releases of the client library, **and** + /// - New values received dynamically, without application changes. + /// + /// Please consult the [Working with enums] section in the user guide for some + /// guidelines. + /// + /// [Working with enums]: https://googleapis.github.io/google-cloud-rust/working_with_enums.html + #[derive(Clone, Debug, PartialEq)] + #[non_exhaustive] + pub enum CloudProvider { + /// Unspecified cloud provider. + Unspecified, + /// Google Cloud Platform. + Gcp, + /// Amazon Web Services. + Aws, + /// Microsoft Azure. + Azure, + /// If set, the enum was initialized with an unknown value. + /// + /// Applications can examine the value using [CloudProvider::value] or + /// [CloudProvider::name]. + UnknownValue(cloud_provider::UnknownValue), + } + + #[doc(hidden)] + pub mod cloud_provider { + #[allow(unused_imports)] + use super::*; + #[derive(Clone, Debug, PartialEq)] + pub struct UnknownValue(pub(crate) wkt::internal::UnknownEnumValue); + } + + impl CloudProvider { + /// Gets the enum value. + /// + /// Returns `None` if the enum contains an unknown value deserialized from + /// the string representation of enums. + pub fn value(&self) -> std::option::Option { + match self { + Self::Unspecified => std::option::Option::Some(0), + Self::Gcp => std::option::Option::Some(1), + Self::Aws => std::option::Option::Some(2), + Self::Azure => std::option::Option::Some(3), + Self::UnknownValue(u) => u.0.value(), + } + } + + /// Gets the enum value as a string. + /// + /// Returns `None` if the enum contains an unknown value deserialized from + /// the integer representation of enums. + pub fn name(&self) -> std::option::Option<&str> { + match self { + Self::Unspecified => std::option::Option::Some("CLOUD_PROVIDER_UNSPECIFIED"), + Self::Gcp => std::option::Option::Some("GCP"), + Self::Aws => std::option::Option::Some("AWS"), + Self::Azure => std::option::Option::Some("AZURE"), + Self::UnknownValue(u) => u.0.name(), + } + } + } + + impl std::default::Default for CloudProvider { + fn default() -> Self { + use std::convert::From; + Self::from(0) + } + } + + impl std::fmt::Display for CloudProvider { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::result::Result<(), std::fmt::Error> { + wkt::internal::display_enum(f, self.name(), self.value()) + } + } + + impl std::convert::From for CloudProvider { + fn from(value: i32) -> Self { + match value { + 0 => Self::Unspecified, + 1 => Self::Gcp, + 2 => Self::Aws, + 3 => Self::Azure, + _ => Self::UnknownValue(cloud_provider::UnknownValue( + wkt::internal::UnknownEnumValue::Integer(value), + )), + } + } + } + + impl std::convert::From<&str> for CloudProvider { + fn from(value: &str) -> Self { + use std::string::ToString; + match value { + "CLOUD_PROVIDER_UNSPECIFIED" => Self::Unspecified, + "GCP" => Self::Gcp, + "AWS" => Self::Aws, + "AZURE" => Self::Azure, + _ => Self::UnknownValue(cloud_provider::UnknownValue( + wkt::internal::UnknownEnumValue::String(value.to_string()), + )), + } + } + } + + impl serde::ser::Serialize for CloudProvider { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + match self { + Self::Unspecified => serializer.serialize_i32(0), + Self::Gcp => serializer.serialize_i32(1), + Self::Aws => serializer.serialize_i32(2), + Self::Azure => serializer.serialize_i32(3), + Self::UnknownValue(u) => u.0.serialize(serializer), + } + } + } + + impl<'de> serde::de::Deserialize<'de> for CloudProvider { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_any(wkt::internal::EnumVisitor::::new( + ".google.cloud.bigquery.v2.ObjectStorageStats.CloudProvider", + )) + } + } +} + /// Statistics for a load job. #[derive(Clone, Default, PartialEq)] #[non_exhaustive] @@ -22889,6 +24218,10 @@ pub struct CopyJobStatistics { /// Output only. Number of logical bytes copied to the destination table. pub copied_logical_bytes: std::option::Option, + /// Output only. Destination region for a cross-region copy job. Not set for + /// in-region copy jobs. + pub remote_destination_region: std::string::String, + pub(crate) _unknown_fields: serde_json::Map, } @@ -22963,6 +24296,21 @@ impl CopyJobStatistics { self.copied_logical_bytes = v.map(|x| x.into()); self } + + /// Sets the value of [remote_destination_region][crate::model::CopyJobStatistics::remote_destination_region]. + /// + /// # Example + /// ```ignore,no_run + /// # use google_cloud_bigquery_v2::model::CopyJobStatistics; + /// let x = CopyJobStatistics::new().set_remote_destination_region("example"); + /// ``` + pub fn set_remote_destination_region>( + mut self, + v: T, + ) -> Self { + self.remote_destination_region = v.into(); + self + } } impl wkt::message::Message for CopyJobStatistics { @@ -23740,6 +25088,12 @@ pub struct JobStatistics { /// the leaf or lowest level reservation group. pub reservation_group_path: std::vec::Vec, + /// Output only. Regions where the global query accesses data. + pub global_query_remote_regions: std::vec::Vec, + + /// Output only. The global query that created this job. + pub parent_global_query_job: std::option::Option, + pub(crate) _unknown_fields: serde_json::Map, } @@ -24283,6 +25637,56 @@ impl JobStatistics { self.reservation_group_path = v.into_iter().map(|i| i.into()).collect(); self } + + /// Sets the value of [global_query_remote_regions][crate::model::JobStatistics::global_query_remote_regions]. + /// + /// # Example + /// ```ignore,no_run + /// # use google_cloud_bigquery_v2::model::JobStatistics; + /// let x = JobStatistics::new().set_global_query_remote_regions(["a", "b", "c"]); + /// ``` + pub fn set_global_query_remote_regions(mut self, v: T) -> Self + where + T: std::iter::IntoIterator, + V: std::convert::Into, + { + use std::iter::Iterator; + self.global_query_remote_regions = v.into_iter().map(|i| i.into()).collect(); + self + } + + /// Sets the value of [parent_global_query_job][crate::model::JobStatistics::parent_global_query_job]. + /// + /// # Example + /// ```ignore,no_run + /// # use google_cloud_bigquery_v2::model::JobStatistics; + /// use google_cloud_bigquery_v2::model::JobReference; + /// let x = JobStatistics::new().set_parent_global_query_job(JobReference::default()/* use setters */); + /// ``` + pub fn set_parent_global_query_job(mut self, v: T) -> Self + where + T: std::convert::Into, + { + self.parent_global_query_job = std::option::Option::Some(v.into()); + self + } + + /// Sets or clears the value of [parent_global_query_job][crate::model::JobStatistics::parent_global_query_job]. + /// + /// # Example + /// ```ignore,no_run + /// # use google_cloud_bigquery_v2::model::JobStatistics; + /// use google_cloud_bigquery_v2::model::JobReference; + /// let x = JobStatistics::new().set_or_clear_parent_global_query_job(Some(JobReference::default()/* use setters */)); + /// let x = JobStatistics::new().set_or_clear_parent_global_query_job(None::); + /// ``` + pub fn set_or_clear_parent_global_query_job(mut self, v: std::option::Option) -> Self + where + T: std::convert::Into, + { + self.parent_global_query_job = v.map(|x| x.into()); + self + } } impl wkt::message::Message for JobStatistics { @@ -25347,6 +26751,16 @@ pub mod partition_skew { /// Output only. Stage id of the skew source stage. pub stage_id: i64, + /// Output only. Median partition output size (in bytes) for this stage. + pub output_bytes_median: i64, + + /// Output only. 95-th percentile of partition output size (in bytes) for + /// this stage. + pub output_bytes_p95: i64, + + /// Output only. Max partition output size (in bytes) for this stage. + pub output_bytes_max: i64, + pub(crate) _unknown_fields: serde_json::Map, } @@ -25367,6 +26781,42 @@ pub mod partition_skew { self.stage_id = v.into(); self } + + /// Sets the value of [output_bytes_median][crate::model::partition_skew::SkewSource::output_bytes_median]. + /// + /// # Example + /// ```ignore,no_run + /// # use google_cloud_bigquery_v2::model::partition_skew::SkewSource; + /// let x = SkewSource::new().set_output_bytes_median(42); + /// ``` + pub fn set_output_bytes_median>(mut self, v: T) -> Self { + self.output_bytes_median = v.into(); + self + } + + /// Sets the value of [output_bytes_p95][crate::model::partition_skew::SkewSource::output_bytes_p95]. + /// + /// # Example + /// ```ignore,no_run + /// # use google_cloud_bigquery_v2::model::partition_skew::SkewSource; + /// let x = SkewSource::new().set_output_bytes_p95(42); + /// ``` + pub fn set_output_bytes_p95>(mut self, v: T) -> Self { + self.output_bytes_p95 = v.into(); + self + } + + /// Sets the value of [output_bytes_max][crate::model::partition_skew::SkewSource::output_bytes_max]. + /// + /// # Example + /// ```ignore,no_run + /// # use google_cloud_bigquery_v2::model::partition_skew::SkewSource; + /// let x = SkewSource::new().set_output_bytes_max(42); + /// ``` + pub fn set_output_bytes_max>(mut self, v: T) -> Self { + self.output_bytes_max = v.into(); + self + } } impl wkt::message::Message for SkewSource { @@ -44442,6 +45892,9 @@ pub mod routine { /// Set if argument_kind == FIXED_TYPE. pub data_type: std::option::Option, + /// Optional. Set if argument_kind == FIXED_TABLE. + pub table_type: std::option::Option, + /// Optional. Whether the argument is an aggregate function parameter. /// Must be Unset for routine types other than AGGREGATE_FUNCTION. /// For AGGREGATE_FUNCTION, if set to false, it is equivalent to adding "NOT @@ -44478,6 +45931,7 @@ pub mod routine { /// use google_cloud_bigquery_v2::model::routine::argument::ArgumentKind; /// let x0 = Argument::new().set_argument_kind(ArgumentKind::FixedType); /// let x1 = Argument::new().set_argument_kind(ArgumentKind::AnyType); + /// let x2 = Argument::new().set_argument_kind(ArgumentKind::FixedTable); /// ``` pub fn set_argument_kind< T: std::convert::Into, @@ -44540,6 +45994,39 @@ pub mod routine { self } + /// Sets the value of [table_type][crate::model::routine::Argument::table_type]. + /// + /// # Example + /// ```ignore,no_run + /// # use google_cloud_bigquery_v2::model::routine::Argument; + /// use google_cloud_bigquery_v2::model::StandardSqlTableType; + /// let x = Argument::new().set_table_type(StandardSqlTableType::default()/* use setters */); + /// ``` + pub fn set_table_type(mut self, v: T) -> Self + where + T: std::convert::Into, + { + self.table_type = std::option::Option::Some(v.into()); + self + } + + /// Sets or clears the value of [table_type][crate::model::routine::Argument::table_type]. + /// + /// # Example + /// ```ignore,no_run + /// # use google_cloud_bigquery_v2::model::routine::Argument; + /// use google_cloud_bigquery_v2::model::StandardSqlTableType; + /// let x = Argument::new().set_or_clear_table_type(Some(StandardSqlTableType::default()/* use setters */)); + /// let x = Argument::new().set_or_clear_table_type(None::); + /// ``` + pub fn set_or_clear_table_type(mut self, v: std::option::Option) -> Self + where + T: std::convert::Into, + { + self.table_type = v.map(|x| x.into()); + self + } + /// Sets the value of [is_aggregate][crate::model::routine::Argument::is_aggregate]. /// /// # Example @@ -44610,6 +46097,10 @@ pub mod routine { FixedType, /// The argument is any type, including struct or array, but not a table. AnyType, + /// The argument is a table with fully specified column names and types. + FixedTable, + /// The argument is any table type. + AnyTable, /// If set, the enum was initialized with an unknown value. /// /// Applications can examine the value using [ArgumentKind::value] or @@ -44635,6 +46126,8 @@ pub mod routine { Self::Unspecified => std::option::Option::Some(0), Self::FixedType => std::option::Option::Some(1), Self::AnyType => std::option::Option::Some(2), + Self::FixedTable => std::option::Option::Some(3), + Self::AnyTable => std::option::Option::Some(4), Self::UnknownValue(u) => u.0.value(), } } @@ -44648,6 +46141,8 @@ pub mod routine { Self::Unspecified => std::option::Option::Some("ARGUMENT_KIND_UNSPECIFIED"), Self::FixedType => std::option::Option::Some("FIXED_TYPE"), Self::AnyType => std::option::Option::Some("ANY_TYPE"), + Self::FixedTable => std::option::Option::Some("FIXED_TABLE"), + Self::AnyTable => std::option::Option::Some("ANY_TABLE"), Self::UnknownValue(u) => u.0.name(), } } @@ -44675,6 +46170,8 @@ pub mod routine { 0 => Self::Unspecified, 1 => Self::FixedType, 2 => Self::AnyType, + 3 => Self::FixedTable, + 4 => Self::AnyTable, _ => Self::UnknownValue(argument_kind::UnknownValue( wkt::internal::UnknownEnumValue::Integer(value), )), @@ -44689,6 +46186,8 @@ pub mod routine { "ARGUMENT_KIND_UNSPECIFIED" => Self::Unspecified, "FIXED_TYPE" => Self::FixedType, "ANY_TYPE" => Self::AnyType, + "FIXED_TABLE" => Self::FixedTable, + "ANY_TABLE" => Self::AnyTable, _ => Self::UnknownValue(argument_kind::UnknownValue( wkt::internal::UnknownEnumValue::String(value.to_string()), )), @@ -44705,6 +46204,8 @@ pub mod routine { Self::Unspecified => serializer.serialize_i32(0), Self::FixedType => serializer.serialize_i32(1), Self::AnyType => serializer.serialize_i32(2), + Self::FixedTable => serializer.serialize_i32(3), + Self::AnyTable => serializer.serialize_i32(4), Self::UnknownValue(u) => u.0.serialize(serializer), } } @@ -45755,8 +47256,10 @@ pub struct ExternalRuntimeOptions { /// Optional. Language runtime version. Example: `python-3.11`. pub runtime_version: std::string::String, - /// Optional. Maximum number of requests that a Cloud Run instance can handle - /// concurrently. If absent or if `0`, a default concurrency is used. + /// Optional. Maximum number of requests that a Python UDF instance can handle + /// concurrently. If absent or if `0`, the default concurrency value is used. + /// For more information, see [Configure container limits for Python + /// UDFs](https://cloud.google.com/bigquery/docs/user-defined-functions-python#configure-container-limits). pub container_request_concurrency: i64, pub(crate) _unknown_fields: serde_json::Map, @@ -48925,7 +50428,7 @@ pub mod table_replication_info { SourceDeleted, /// Source revoked replication permissions. PermissionDenied, - /// Source configuration doesn’t allow replication. + /// Source configuration doesn't allow replication. UnsupportedConfiguration, /// If set, the enum was initialized with an unknown value. /// @@ -53488,21 +54991,19 @@ pub struct TableFieldSchema { /// Optional. Specifies the data governance tags on this field. This field /// works with other column-level security fields as follows: /// - /// - Precedence: If a data governance tag is attached to a column, it takes - /// precedence over the policy tag attached to the column. - /// However, if a data policy is attached to a column, it takes precedence - /// over the data governance tag. - /// - /// - Patching behavior (how this field behaves during a `Table.patch` schema - /// update): - /// - /// - Unset: If the `data_governance_tags_info` field is omitted - /// from the update request, the existing tags on the column are preserved. - /// - Empty Field: To clear data governance tags from a column, send the - /// `data_governance_tags_info` field as an empty object. This will remove - /// all tags from the column. - /// - Updating tags: To replace existing tag, send the field with the - /// new tag. + /// * **Precedence**: If a data governance tag is attached to a column, it + /// takes precedence over the policy tag attached to the column. However, + /// if a data policy is attached to a column, it takes precedence over the + /// data governance tag. + /// * **Patching behavior**: Describes how this field behaves during a + /// `Table.patch` schema update: + /// * **Unset**: If the `data_governance_tags_info` field is omitted from + /// the update request, the existing tags on the column are preserved. + /// * **Empty Field**: To clear data governance tags from a column, send + /// the `data_governance_tags_info` field as an empty object. This + /// removes all tags from the column. + /// * **Updating tags**: To replace an existing tag, send the field with + /// the new tag. pub data_governance_tags_info: std::option::Option, @@ -54125,13 +55626,13 @@ pub mod table_field_schema { /// Optional. The data governance tags added to this field are used for /// field-level access control. Only one data governance tag is currently /// supported on a field. Tag keys are globally unique. Tag key is expected - /// to be in the namespaced format, for example "123456789012/pii" where - /// 123456789012 is the ID of the parent organization or project resource for + /// to be in the namespaced format, for example "parent-id/pii" where + /// parent-id is the ID of the parent organization or project resource for /// this tag key. Tag value is expected to be the short name, for example /// "sensitive". See [Tag /// definitions](https://cloud.google.com/iam/docs/tags-access-control#definitions) /// for more details. For example: - /// "123456789012/pii": "sensitive", + /// "parent-id/pii": "sensitive", /// "myProject/cost_center": "sales" pub data_governance_tags: std::collections::HashMap, @@ -54215,8 +55716,8 @@ pub mod table_field_schema { #[derive(Clone, Default, PartialEq)] #[non_exhaustive] pub struct GeneratedExpressionInfo { - /// Optional. The generation expression (e.g. AI.EMBED(...)) used to - /// generated the field. + /// Optional. The generation expression (e.g. AI.EMBED(...)) used to generate + /// the field. pub generation_expression: std::option::Option, /// Optional. Whether the column generation is done asynchronously. @@ -54344,8 +55845,7 @@ pub mod table_field_schema { pub generated_mode: std::option::Option, - /// Captures the metadata for the generated column. Could be either an - /// identity column or a generated column. + /// Captures the metadata for the generated column. pub definition: std::option::Option, @@ -54616,8 +56116,7 @@ pub mod table_field_schema { } } - /// Captures the metadata for the generated column. Could be either an - /// identity column or a generated column. + /// Captures the metadata for the generated column. #[derive(Clone, Debug, PartialEq)] #[non_exhaustive] pub enum Definition { diff --git a/src/generated/cloud/bigquery/v2/src/model/debug.rs b/src/generated/cloud/bigquery/v2/src/model/debug.rs index c33c9a1ec0..147ee75bff 100644 --- a/src/generated/cloud/bigquery/v2/src/model/debug.rs +++ b/src/generated/cloud/bigquery/v2/src/model/debug.rs @@ -17,6 +17,40 @@ #[allow(unused_imports)] use super::*; +impl std::fmt::Debug for super::ArrowSchema { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut debug_struct = f.debug_struct("ArrowSchema"); + debug_struct.field("serialized_schema", &self.serialized_schema); + if !self._unknown_fields.is_empty() { + debug_struct.field("_unknown_fields", &self._unknown_fields); + } + debug_struct.finish() + } +} + +impl std::fmt::Debug for super::ArrowRecordBatch { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut debug_struct = f.debug_struct("ArrowRecordBatch"); + debug_struct.field("serialized_record_batch", &self.serialized_record_batch); + if !self._unknown_fields.is_empty() { + debug_struct.field("_unknown_fields", &self._unknown_fields); + } + debug_struct.finish() + } +} + +impl std::fmt::Debug for super::ArrowSerializationOptions { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut debug_struct = f.debug_struct("ArrowSerializationOptions"); + debug_struct.field("buffer_compression", &self.buffer_compression); + debug_struct.field("picos_timestamp_precision", &self.picos_timestamp_precision); + if !self._unknown_fields.is_empty() { + debug_struct.field("_unknown_fields", &self._unknown_fields); + } + debug_struct.finish() + } +} + impl std::fmt::Debug for super::BigLakeConfiguration { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let mut debug_struct = f.debug_struct("BigLakeConfiguration"); @@ -882,6 +916,11 @@ impl std::fmt::Debug for super::QueryRequest { debug_struct.field("job_creation_mode", &self.job_creation_mode); debug_struct.field("reservation", &self.reservation); debug_struct.field("write_incremental_results", &self.write_incremental_results); + debug_struct.field("query_results_format", &self.query_results_format); + debug_struct.field( + "results_format_serialization_options", + &self.results_format_serialization_options, + ); if !self._unknown_fields.is_empty() { debug_struct.field("_unknown_fields", &self._unknown_fields); } @@ -913,6 +952,10 @@ impl std::fmt::Debug for super::QueryResponse { debug_struct.field("creation_time", &self.creation_time); debug_struct.field("start_time", &self.start_time); debug_struct.field("end_time", &self.end_time); + debug_struct.field("page_row_count", &self.page_row_count); + debug_struct.field("statement_type", &self.statement_type); + debug_struct.field("results_schema", &self.results_schema); + debug_struct.field("results", &self.results); if !self._unknown_fields.is_empty() { debug_struct.field("_unknown_fields", &self._unknown_fields); } @@ -1503,6 +1546,20 @@ impl std::fmt::Debug for super::JobStatistics2 { debug_struct.field("metadata_cache_statistics", &self.metadata_cache_statistics); debug_struct.field("incremental_result_stats", &self.incremental_result_stats); debug_struct.field("gen_ai_stats", &self.gen_ai_stats); + debug_struct.field("object_storage_stats", &self.object_storage_stats); + if !self._unknown_fields.is_empty() { + debug_struct.field("_unknown_fields", &self._unknown_fields); + } + debug_struct.finish() + } +} + +impl std::fmt::Debug for super::ObjectStorageStats { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut debug_struct = f.debug_struct("ObjectStorageStats"); + debug_struct.field("cloud_provider", &self.cloud_provider); + debug_struct.field("object_storage_bytes_read", &self.object_storage_bytes_read); + debug_struct.field("cache_bytes_read", &self.cache_bytes_read); if !self._unknown_fields.is_empty() { debug_struct.field("_unknown_fields", &self._unknown_fields); } @@ -1547,6 +1604,7 @@ impl std::fmt::Debug for super::CopyJobStatistics { let mut debug_struct = f.debug_struct("CopyJobStatistics"); debug_struct.field("copied_rows", &self.copied_rows); debug_struct.field("copied_logical_bytes", &self.copied_logical_bytes); + debug_struct.field("remote_destination_region", &self.remote_destination_region); if !self._unknown_fields.is_empty() { debug_struct.field("_unknown_fields", &self._unknown_fields); } @@ -1653,6 +1711,11 @@ impl std::fmt::Debug for super::JobStatistics { ); debug_struct.field("edition", &self.edition); debug_struct.field("reservation_group_path", &self.reservation_group_path); + debug_struct.field( + "global_query_remote_regions", + &self.global_query_remote_regions, + ); + debug_struct.field("parent_global_query_job", &self.parent_global_query_job); if !self._unknown_fields.is_empty() { debug_struct.field("_unknown_fields", &self._unknown_fields); } @@ -1783,6 +1846,9 @@ impl std::fmt::Debug for super::partition_skew::SkewSource { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let mut debug_struct = f.debug_struct("SkewSource"); debug_struct.field("stage_id", &self.stage_id); + debug_struct.field("output_bytes_median", &self.output_bytes_median); + debug_struct.field("output_bytes_p95", &self.output_bytes_p95); + debug_struct.field("output_bytes_max", &self.output_bytes_max); if !self._unknown_fields.is_empty() { debug_struct.field("_unknown_fields", &self._unknown_fields); } @@ -3171,6 +3237,7 @@ impl std::fmt::Debug for super::routine::Argument { debug_struct.field("argument_kind", &self.argument_kind); debug_struct.field("mode", &self.mode); debug_struct.field("data_type", &self.data_type); + debug_struct.field("table_type", &self.table_type); debug_struct.field("is_aggregate", &self.is_aggregate); if !self._unknown_fields.is_empty() { debug_struct.field("_unknown_fields", &self._unknown_fields); diff --git a/src/generated/cloud/bigquery/v2/src/model/deserialize.rs b/src/generated/cloud/bigquery/v2/src/model/deserialize.rs index 21e9df9915..6b864fa92c 100644 --- a/src/generated/cloud/bigquery/v2/src/model/deserialize.rs +++ b/src/generated/cloud/bigquery/v2/src/model/deserialize.rs @@ -17,6 +17,286 @@ #[allow(unused_imports)] use super::*; +#[doc(hidden)] +impl<'de> serde::de::Deserialize<'de> for super::ArrowSchema { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + #[allow(non_camel_case_types)] + #[doc(hidden)] + #[derive(PartialEq, Eq, Hash)] + enum __FieldTag { + __serialized_schema, + Unknown(std::string::String), + } + impl<'de> serde::de::Deserialize<'de> for __FieldTag { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct Visitor; + impl<'de> serde::de::Visitor<'de> for Visitor { + type Value = __FieldTag; + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a field name for ArrowSchema") + } + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + use std::result::Result::Ok; + use std::string::ToString; + match value { + "serializedSchema" => Ok(__FieldTag::__serialized_schema), + "serialized_schema" => Ok(__FieldTag::__serialized_schema), + _ => Ok(__FieldTag::Unknown(value.to_string())), + } + } + } + deserializer.deserialize_identifier(Visitor) + } + } + struct Visitor; + impl<'de> serde::de::Visitor<'de> for Visitor { + type Value = super::ArrowSchema; + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("struct ArrowSchema") + } + fn visit_map(self, mut map: A) -> std::result::Result + where + A: serde::de::MapAccess<'de>, + { + #[allow(unused_imports)] + use serde::de::Error; + use std::option::Option::Some; + let mut fields = std::collections::HashSet::new(); + let mut result = Self::Value::new(); + while let Some(tag) = map.next_key::<__FieldTag>()? { + #[allow(clippy::match_single_binding)] + match tag { + __FieldTag::__serialized_schema => { + if !fields.insert(__FieldTag::__serialized_schema) { + return std::result::Result::Err(A::Error::duplicate_field( + "multiple values for serialized_schema", + )); + } + struct __With(std::option::Option<::bytes::Bytes>); + impl<'de> serde::de::Deserialize<'de> for __With { + fn deserialize( + deserializer: D, + ) -> std::result::Result + where + D: serde::de::Deserializer<'de>, + { + serde_with::As::< std::option::Option >::deserialize(deserializer).map(__With) + } + } + result.serialized_schema = + map.next_value::<__With>()?.0.unwrap_or_default(); + } + __FieldTag::Unknown(key) => { + let value = map.next_value::()?; + result._unknown_fields.insert(key, value); + } + } + } + std::result::Result::Ok(result) + } + } + deserializer.deserialize_any(Visitor) + } +} + +#[doc(hidden)] +impl<'de> serde::de::Deserialize<'de> for super::ArrowRecordBatch { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + #[allow(non_camel_case_types)] + #[doc(hidden)] + #[derive(PartialEq, Eq, Hash)] + enum __FieldTag { + __serialized_record_batch, + Unknown(std::string::String), + } + impl<'de> serde::de::Deserialize<'de> for __FieldTag { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct Visitor; + impl<'de> serde::de::Visitor<'de> for Visitor { + type Value = __FieldTag; + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a field name for ArrowRecordBatch") + } + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + use std::result::Result::Ok; + use std::string::ToString; + match value { + "serializedRecordBatch" => Ok(__FieldTag::__serialized_record_batch), + "serialized_record_batch" => Ok(__FieldTag::__serialized_record_batch), + _ => Ok(__FieldTag::Unknown(value.to_string())), + } + } + } + deserializer.deserialize_identifier(Visitor) + } + } + struct Visitor; + impl<'de> serde::de::Visitor<'de> for Visitor { + type Value = super::ArrowRecordBatch; + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("struct ArrowRecordBatch") + } + fn visit_map(self, mut map: A) -> std::result::Result + where + A: serde::de::MapAccess<'de>, + { + #[allow(unused_imports)] + use serde::de::Error; + use std::option::Option::Some; + let mut fields = std::collections::HashSet::new(); + let mut result = Self::Value::new(); + while let Some(tag) = map.next_key::<__FieldTag>()? { + #[allow(clippy::match_single_binding)] + match tag { + __FieldTag::__serialized_record_batch => { + if !fields.insert(__FieldTag::__serialized_record_batch) { + return std::result::Result::Err(A::Error::duplicate_field( + "multiple values for serialized_record_batch", + )); + } + struct __With(std::option::Option<::bytes::Bytes>); + impl<'de> serde::de::Deserialize<'de> for __With { + fn deserialize( + deserializer: D, + ) -> std::result::Result + where + D: serde::de::Deserializer<'de>, + { + serde_with::As::< std::option::Option >::deserialize(deserializer).map(__With) + } + } + result.serialized_record_batch = + map.next_value::<__With>()?.0.unwrap_or_default(); + } + __FieldTag::Unknown(key) => { + let value = map.next_value::()?; + result._unknown_fields.insert(key, value); + } + } + } + std::result::Result::Ok(result) + } + } + deserializer.deserialize_any(Visitor) + } +} + +#[doc(hidden)] +impl<'de> serde::de::Deserialize<'de> for super::ArrowSerializationOptions { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + #[allow(non_camel_case_types)] + #[doc(hidden)] + #[derive(PartialEq, Eq, Hash)] + enum __FieldTag { + __buffer_compression, + __picos_timestamp_precision, + Unknown(std::string::String), + } + impl<'de> serde::de::Deserialize<'de> for __FieldTag { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct Visitor; + impl<'de> serde::de::Visitor<'de> for Visitor { + type Value = __FieldTag; + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a field name for ArrowSerializationOptions") + } + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + use std::result::Result::Ok; + use std::string::ToString; + match value { + "bufferCompression" => Ok(__FieldTag::__buffer_compression), + "buffer_compression" => Ok(__FieldTag::__buffer_compression), + "picosTimestampPrecision" => { + Ok(__FieldTag::__picos_timestamp_precision) + } + "picos_timestamp_precision" => { + Ok(__FieldTag::__picos_timestamp_precision) + } + _ => Ok(__FieldTag::Unknown(value.to_string())), + } + } + } + deserializer.deserialize_identifier(Visitor) + } + } + struct Visitor; + impl<'de> serde::de::Visitor<'de> for Visitor { + type Value = super::ArrowSerializationOptions; + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("struct ArrowSerializationOptions") + } + fn visit_map(self, mut map: A) -> std::result::Result + where + A: serde::de::MapAccess<'de>, + { + #[allow(unused_imports)] + use serde::de::Error; + use std::option::Option::Some; + let mut fields = std::collections::HashSet::new(); + let mut result = Self::Value::new(); + while let Some(tag) = map.next_key::<__FieldTag>()? { + #[allow(clippy::match_single_binding)] + match tag { + __FieldTag::__buffer_compression => { + if !fields.insert(__FieldTag::__buffer_compression) { + return std::result::Result::Err(A::Error::duplicate_field( + "multiple values for buffer_compression", + )); + } + result.buffer_compression = map + .next_value::>()? + .unwrap_or_default(); + } + __FieldTag::__picos_timestamp_precision => { + if !fields.insert(__FieldTag::__picos_timestamp_precision) { + return std::result::Result::Err(A::Error::duplicate_field( + "multiple values for picos_timestamp_precision", + )); + } + result.picos_timestamp_precision = map.next_value::>()?.unwrap_or_default(); + } + __FieldTag::Unknown(key) => { + let value = map.next_value::()?; + result._unknown_fields.insert(key, value); + } + } + } + std::result::Result::Ok(result) + } + } + deserializer.deserialize_any(Visitor) + } +} + #[doc(hidden)] impl<'de> serde::de::Deserialize<'de> for super::BigLakeConfiguration { fn deserialize(deserializer: D) -> std::result::Result @@ -7324,6 +7604,8 @@ impl<'de> serde::de::Deserialize<'de> for super::QueryRequest { __job_creation_mode, __reservation, __write_incremental_results, + __query_results_format, + __arrow_serialization_options, Unknown(std::string::String), } impl<'de> serde::de::Deserialize<'de> for __FieldTag { @@ -7393,6 +7675,14 @@ impl<'de> serde::de::Deserialize<'de> for super::QueryRequest { "write_incremental_results" => { Ok(__FieldTag::__write_incremental_results) } + "queryResultsFormat" => Ok(__FieldTag::__query_results_format), + "query_results_format" => Ok(__FieldTag::__query_results_format), + "arrowSerializationOptions" => { + Ok(__FieldTag::__arrow_serialization_options) + } + "arrow_serialization_options" => { + Ok(__FieldTag::__arrow_serialization_options) + } _ => Ok(__FieldTag::Unknown(value.to_string())), } } @@ -7699,6 +7989,35 @@ impl<'de> serde::de::Deserialize<'de> for super::QueryRequest { .next_value::>()? .unwrap_or_default(); } + __FieldTag::__query_results_format => { + if !fields.insert(__FieldTag::__query_results_format) { + return std::result::Result::Err(A::Error::duplicate_field( + "multiple values for query_results_format", + )); + } + result.query_results_format = map + .next_value::>()? + .unwrap_or_default(); + } + __FieldTag::__arrow_serialization_options => { + if !fields.insert(__FieldTag::__arrow_serialization_options) { + return std::result::Result::Err(A::Error::duplicate_field( + "multiple values for arrow_serialization_options", + )); + } + if result.results_format_serialization_options.is_some() { + return std::result::Result::Err(A::Error::duplicate_field( + "multiple values for `results_format_serialization_options`, a oneof with full ID .google.cloud.bigquery.v2.QueryRequest.arrow_serialization_options, latest field was arrowSerializationOptions", + )); + } + result.results_format_serialization_options = std::option::Option::Some( + crate::model::query_request::ResultsFormatSerializationOptions::ArrowSerializationOptions( + map.next_value::>>()?.unwrap_or_default() + ), + ); + } __FieldTag::Unknown(key) => { let value = map.next_value::()?; result._unknown_fields.insert(key, value); @@ -7743,6 +8062,10 @@ impl<'de> serde::de::Deserialize<'de> for super::QueryResponse { __creation_time, __start_time, __end_time, + __arrow_schema, + __arrow_record_batch, + __page_row_count, + __statement_type, Unknown(std::string::String), } impl<'de> serde::de::Deserialize<'de> for __FieldTag { @@ -7800,6 +8123,14 @@ impl<'de> serde::de::Deserialize<'de> for super::QueryResponse { "start_time" => Ok(__FieldTag::__start_time), "endTime" => Ok(__FieldTag::__end_time), "end_time" => Ok(__FieldTag::__end_time), + "arrowSchema" => Ok(__FieldTag::__arrow_schema), + "arrow_schema" => Ok(__FieldTag::__arrow_schema), + "arrowRecordBatch" => Ok(__FieldTag::__arrow_record_batch), + "arrow_record_batch" => Ok(__FieldTag::__arrow_record_batch), + "pageRowCount" => Ok(__FieldTag::__page_row_count), + "page_row_count" => Ok(__FieldTag::__page_row_count), + "statementType" => Ok(__FieldTag::__statement_type), + "statement_type" => Ok(__FieldTag::__statement_type), _ => Ok(__FieldTag::Unknown(value.to_string())), } } @@ -8099,6 +8430,76 @@ impl<'de> serde::de::Deserialize<'de> for super::QueryResponse { } result.end_time = map.next_value::<__With>()?.0; } + __FieldTag::__arrow_schema => { + if !fields.insert(__FieldTag::__arrow_schema) { + return std::result::Result::Err(A::Error::duplicate_field( + "multiple values for arrow_schema", + )); + } + if result.results_schema.is_some() { + return std::result::Result::Err(A::Error::duplicate_field( + "multiple values for `results_schema`, a oneof with full ID .google.cloud.bigquery.v2.QueryResponse.arrow_schema, latest field was arrowSchema", + )); + } + result.results_schema = std::option::Option::Some( + crate::model::query_response::ResultsSchema::ArrowSchema( + map.next_value::, + >>()? + .unwrap_or_default(), + ), + ); + } + __FieldTag::__arrow_record_batch => { + if !fields.insert(__FieldTag::__arrow_record_batch) { + return std::result::Result::Err(A::Error::duplicate_field( + "multiple values for arrow_record_batch", + )); + } + if result.results.is_some() { + return std::result::Result::Err(A::Error::duplicate_field( + "multiple values for `results`, a oneof with full ID .google.cloud.bigquery.v2.QueryResponse.arrow_record_batch, latest field was arrowRecordBatch", + )); + } + result.results = std::option::Option::Some( + crate::model::query_response::Results::ArrowRecordBatch( + map.next_value::, + >>()? + .unwrap_or_default(), + ), + ); + } + __FieldTag::__page_row_count => { + if !fields.insert(__FieldTag::__page_row_count) { + return std::result::Result::Err(A::Error::duplicate_field( + "multiple values for page_row_count", + )); + } + struct __With(std::option::Option); + impl<'de> serde::de::Deserialize<'de> for __With { + fn deserialize( + deserializer: D, + ) -> std::result::Result + where + D: serde::de::Deserializer<'de>, + { + serde_with::As::< std::option::Option >::deserialize(deserializer).map(__With) + } + } + result.page_row_count = + map.next_value::<__With>()?.0.unwrap_or_default(); + } + __FieldTag::__statement_type => { + if !fields.insert(__FieldTag::__statement_type) { + return std::result::Result::Err(A::Error::duplicate_field( + "multiple values for statement_type", + )); + } + result.statement_type = map + .next_value::>()? + .unwrap_or_default(); + } __FieldTag::Unknown(key) => { let value = map.next_value::()?; result._unknown_fields.insert(key, value); @@ -13112,6 +13513,7 @@ impl<'de> serde::de::Deserialize<'de> for super::JobStatistics2 { __metadata_cache_statistics, __incremental_result_stats, __gen_ai_stats, + __object_storage_stats, Unknown(std::string::String), } impl<'de> serde::de::Deserialize<'de> for __FieldTag { @@ -13262,6 +13664,8 @@ impl<'de> serde::de::Deserialize<'de> for super::JobStatistics2 { } "genAiStats" => Ok(__FieldTag::__gen_ai_stats), "gen_ai_stats" => Ok(__FieldTag::__gen_ai_stats), + "objectStorageStats" => Ok(__FieldTag::__object_storage_stats), + "object_storage_stats" => Ok(__FieldTag::__object_storage_stats), _ => Ok(__FieldTag::Unknown(value.to_string())), } } @@ -13817,6 +14221,145 @@ impl<'de> serde::de::Deserialize<'de> for super::JobStatistics2 { result.gen_ai_stats = map.next_value::>()?; } + __FieldTag::__object_storage_stats => { + if !fields.insert(__FieldTag::__object_storage_stats) { + return std::result::Result::Err(A::Error::duplicate_field( + "multiple values for object_storage_stats", + )); + } + result.object_storage_stats = map + .next_value::, + >>()? + .unwrap_or_default(); + } + __FieldTag::Unknown(key) => { + let value = map.next_value::()?; + result._unknown_fields.insert(key, value); + } + } + } + std::result::Result::Ok(result) + } + } + deserializer.deserialize_any(Visitor) + } +} + +#[doc(hidden)] +impl<'de> serde::de::Deserialize<'de> for super::ObjectStorageStats { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + #[allow(non_camel_case_types)] + #[doc(hidden)] + #[derive(PartialEq, Eq, Hash)] + enum __FieldTag { + __cloud_provider, + __object_storage_bytes_read, + __cache_bytes_read, + Unknown(std::string::String), + } + impl<'de> serde::de::Deserialize<'de> for __FieldTag { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct Visitor; + impl<'de> serde::de::Visitor<'de> for Visitor { + type Value = __FieldTag; + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a field name for ObjectStorageStats") + } + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + use std::result::Result::Ok; + use std::string::ToString; + match value { + "cloudProvider" => Ok(__FieldTag::__cloud_provider), + "cloud_provider" => Ok(__FieldTag::__cloud_provider), + "objectStorageBytesRead" => Ok(__FieldTag::__object_storage_bytes_read), + "object_storage_bytes_read" => { + Ok(__FieldTag::__object_storage_bytes_read) + } + "cacheBytesRead" => Ok(__FieldTag::__cache_bytes_read), + "cache_bytes_read" => Ok(__FieldTag::__cache_bytes_read), + _ => Ok(__FieldTag::Unknown(value.to_string())), + } + } + } + deserializer.deserialize_identifier(Visitor) + } + } + struct Visitor; + impl<'de> serde::de::Visitor<'de> for Visitor { + type Value = super::ObjectStorageStats; + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("struct ObjectStorageStats") + } + fn visit_map(self, mut map: A) -> std::result::Result + where + A: serde::de::MapAccess<'de>, + { + #[allow(unused_imports)] + use serde::de::Error; + use std::option::Option::Some; + let mut fields = std::collections::HashSet::new(); + let mut result = Self::Value::new(); + while let Some(tag) = map.next_key::<__FieldTag>()? { + #[allow(clippy::match_single_binding)] + match tag { + __FieldTag::__cloud_provider => { + if !fields.insert(__FieldTag::__cloud_provider) { + return std::result::Result::Err(A::Error::duplicate_field( + "multiple values for cloud_provider", + )); + } + result.cloud_provider = map.next_value::>()?; + } + __FieldTag::__object_storage_bytes_read => { + if !fields.insert(__FieldTag::__object_storage_bytes_read) { + return std::result::Result::Err(A::Error::duplicate_field( + "multiple values for object_storage_bytes_read", + )); + } + struct __With(std::option::Option); + impl<'de> serde::de::Deserialize<'de> for __With { + fn deserialize( + deserializer: D, + ) -> std::result::Result + where + D: serde::de::Deserializer<'de>, + { + serde_with::As::< std::option::Option >::deserialize(deserializer).map(__With) + } + } + result.object_storage_bytes_read = map.next_value::<__With>()?.0; + } + __FieldTag::__cache_bytes_read => { + if !fields.insert(__FieldTag::__cache_bytes_read) { + return std::result::Result::Err(A::Error::duplicate_field( + "multiple values for cache_bytes_read", + )); + } + struct __With(std::option::Option); + impl<'de> serde::de::Deserialize<'de> for __With { + fn deserialize( + deserializer: D, + ) -> std::result::Result + where + D: serde::de::Deserializer<'de>, + { + serde_with::As::< std::option::Option >::deserialize(deserializer).map(__With) + } + } + result.cache_bytes_read = map.next_value::<__With>()?.0; + } __FieldTag::Unknown(key) => { let value = map.next_value::()?; result._unknown_fields.insert(key, value); @@ -14168,6 +14711,7 @@ impl<'de> serde::de::Deserialize<'de> for super::CopyJobStatistics { enum __FieldTag { __copied_rows, __copied_logical_bytes, + __remote_destination_region, Unknown(std::string::String), } impl<'de> serde::de::Deserialize<'de> for __FieldTag { @@ -14192,6 +14736,12 @@ impl<'de> serde::de::Deserialize<'de> for super::CopyJobStatistics { "copied_rows" => Ok(__FieldTag::__copied_rows), "copiedLogicalBytes" => Ok(__FieldTag::__copied_logical_bytes), "copied_logical_bytes" => Ok(__FieldTag::__copied_logical_bytes), + "remoteDestinationRegion" => { + Ok(__FieldTag::__remote_destination_region) + } + "remote_destination_region" => { + Ok(__FieldTag::__remote_destination_region) + } _ => Ok(__FieldTag::Unknown(value.to_string())), } } @@ -14255,6 +14805,16 @@ impl<'de> serde::de::Deserialize<'de> for super::CopyJobStatistics { } result.copied_logical_bytes = map.next_value::<__With>()?.0; } + __FieldTag::__remote_destination_region => { + if !fields.insert(__FieldTag::__remote_destination_region) { + return std::result::Result::Err(A::Error::duplicate_field( + "multiple values for remote_destination_region", + )); + } + result.remote_destination_region = map + .next_value::>()? + .unwrap_or_default(); + } __FieldTag::Unknown(key) => { let value = map.next_value::()?; result._unknown_fields.insert(key, value); @@ -14895,6 +15455,8 @@ impl<'de> serde::de::Deserialize<'de> for super::JobStatistics { __final_execution_duration_ms, __edition, __reservation_group_path, + __global_query_remote_regions, + __parent_global_query_job, Unknown(std::string::String), } impl<'de> serde::de::Deserialize<'de> for __FieldTag { @@ -14961,6 +15523,14 @@ impl<'de> serde::de::Deserialize<'de> for super::JobStatistics { "edition" => Ok(__FieldTag::__edition), "reservationGroupPath" => Ok(__FieldTag::__reservation_group_path), "reservation_group_path" => Ok(__FieldTag::__reservation_group_path), + "globalQueryRemoteRegions" => { + Ok(__FieldTag::__global_query_remote_regions) + } + "global_query_remote_regions" => { + Ok(__FieldTag::__global_query_remote_regions) + } + "parentGlobalQueryJob" => Ok(__FieldTag::__parent_global_query_job), + "parent_global_query_job" => Ok(__FieldTag::__parent_global_query_job), _ => Ok(__FieldTag::Unknown(value.to_string())), } } @@ -15272,6 +15842,23 @@ impl<'de> serde::de::Deserialize<'de> for super::JobStatistics { } result.reservation_group_path = map.next_value::>>()?.unwrap_or_default(); } + __FieldTag::__global_query_remote_regions => { + if !fields.insert(__FieldTag::__global_query_remote_regions) { + return std::result::Result::Err(A::Error::duplicate_field( + "multiple values for global_query_remote_regions", + )); + } + result.global_query_remote_regions = map.next_value::>>()?.unwrap_or_default(); + } + __FieldTag::__parent_global_query_job => { + if !fields.insert(__FieldTag::__parent_global_query_job) { + return std::result::Result::Err(A::Error::duplicate_field( + "multiple values for parent_global_query_job", + )); + } + result.parent_global_query_job = map + .next_value::>()?; + } __FieldTag::Unknown(key) => { let value = map.next_value::()?; result._unknown_fields.insert(key, value); @@ -16284,6 +16871,9 @@ impl<'de> serde::de::Deserialize<'de> for super::partition_skew::SkewSource { #[derive(PartialEq, Eq, Hash)] enum __FieldTag { __stage_id, + __output_bytes_median, + __output_bytes_p95, + __output_bytes_max, Unknown(std::string::String), } impl<'de> serde::de::Deserialize<'de> for __FieldTag { @@ -16306,6 +16896,12 @@ impl<'de> serde::de::Deserialize<'de> for super::partition_skew::SkewSource { match value { "stageId" => Ok(__FieldTag::__stage_id), "stage_id" => Ok(__FieldTag::__stage_id), + "outputBytesMedian" => Ok(__FieldTag::__output_bytes_median), + "output_bytes_median" => Ok(__FieldTag::__output_bytes_median), + "outputBytesP95" => Ok(__FieldTag::__output_bytes_p95), + "output_bytes_p95" => Ok(__FieldTag::__output_bytes_p95), + "outputBytesMax" => Ok(__FieldTag::__output_bytes_max), + "output_bytes_max" => Ok(__FieldTag::__output_bytes_max), _ => Ok(__FieldTag::Unknown(value.to_string())), } } @@ -16350,6 +16946,66 @@ impl<'de> serde::de::Deserialize<'de> for super::partition_skew::SkewSource { } result.stage_id = map.next_value::<__With>()?.0.unwrap_or_default(); } + __FieldTag::__output_bytes_median => { + if !fields.insert(__FieldTag::__output_bytes_median) { + return std::result::Result::Err(A::Error::duplicate_field( + "multiple values for output_bytes_median", + )); + } + struct __With(std::option::Option); + impl<'de> serde::de::Deserialize<'de> for __With { + fn deserialize( + deserializer: D, + ) -> std::result::Result + where + D: serde::de::Deserializer<'de>, + { + serde_with::As::< std::option::Option >::deserialize(deserializer).map(__With) + } + } + result.output_bytes_median = + map.next_value::<__With>()?.0.unwrap_or_default(); + } + __FieldTag::__output_bytes_p95 => { + if !fields.insert(__FieldTag::__output_bytes_p95) { + return std::result::Result::Err(A::Error::duplicate_field( + "multiple values for output_bytes_p95", + )); + } + struct __With(std::option::Option); + impl<'de> serde::de::Deserialize<'de> for __With { + fn deserialize( + deserializer: D, + ) -> std::result::Result + where + D: serde::de::Deserializer<'de>, + { + serde_with::As::< std::option::Option >::deserialize(deserializer).map(__With) + } + } + result.output_bytes_p95 = + map.next_value::<__With>()?.0.unwrap_or_default(); + } + __FieldTag::__output_bytes_max => { + if !fields.insert(__FieldTag::__output_bytes_max) { + return std::result::Result::Err(A::Error::duplicate_field( + "multiple values for output_bytes_max", + )); + } + struct __With(std::option::Option); + impl<'de> serde::de::Deserialize<'de> for __With { + fn deserialize( + deserializer: D, + ) -> std::result::Result + where + D: serde::de::Deserializer<'de>, + { + serde_with::As::< std::option::Option >::deserialize(deserializer).map(__With) + } + } + result.output_bytes_max = + map.next_value::<__With>()?.0.unwrap_or_default(); + } __FieldTag::Unknown(key) => { let value = map.next_value::()?; result._unknown_fields.insert(key, value); @@ -29148,6 +29804,7 @@ impl<'de> serde::de::Deserialize<'de> for super::routine::Argument { __argument_kind, __mode, __data_type, + __table_type, __is_aggregate, Unknown(std::string::String), } @@ -29175,6 +29832,8 @@ impl<'de> serde::de::Deserialize<'de> for super::routine::Argument { "mode" => Ok(__FieldTag::__mode), "dataType" => Ok(__FieldTag::__data_type), "data_type" => Ok(__FieldTag::__data_type), + "tableType" => Ok(__FieldTag::__table_type), + "table_type" => Ok(__FieldTag::__table_type), "isAggregate" => Ok(__FieldTag::__is_aggregate), "is_aggregate" => Ok(__FieldTag::__is_aggregate), _ => Ok(__FieldTag::Unknown(value.to_string())), @@ -29241,6 +29900,15 @@ impl<'de> serde::de::Deserialize<'de> for super::routine::Argument { result.data_type = map.next_value::>()? ; } + __FieldTag::__table_type => { + if !fields.insert(__FieldTag::__table_type) { + return std::result::Result::Err(A::Error::duplicate_field( + "multiple values for table_type", + )); + } + result.table_type = map.next_value::>()? + ; + } __FieldTag::__is_aggregate => { if !fields.insert(__FieldTag::__is_aggregate) { return std::result::Result::Err(A::Error::duplicate_field( diff --git a/src/generated/cloud/bigquery/v2/src/model/serialize.rs b/src/generated/cloud/bigquery/v2/src/model/serialize.rs index 79ba108b2b..e219be7b03 100644 --- a/src/generated/cloud/bigquery/v2/src/model/serialize.rs +++ b/src/generated/cloud/bigquery/v2/src/model/serialize.rs @@ -17,6 +17,96 @@ #[allow(unused_imports)] use super::*; +#[doc(hidden)] +impl serde::ser::Serialize for super::ArrowSchema { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::ser::Serializer, + { + use serde::ser::SerializeMap; + #[allow(unused_imports)] + use std::option::Option::Some; + let mut state = serializer.serialize_map(std::option::Option::None)?; + if !self.serialized_schema.is_empty() { + struct __With<'a>(&'a ::bytes::Bytes); + impl<'a> serde::ser::Serialize for __With<'a> { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::ser::Serializer, + { + serde_with::As::::serialize(self.0, serializer) + } + } + state.serialize_entry("serializedSchema", &__With(&self.serialized_schema))?; + } + if !self._unknown_fields.is_empty() { + for (key, value) in self._unknown_fields.iter() { + state.serialize_entry(key, &value)?; + } + } + state.end() + } +} + +#[doc(hidden)] +impl serde::ser::Serialize for super::ArrowRecordBatch { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::ser::Serializer, + { + use serde::ser::SerializeMap; + #[allow(unused_imports)] + use std::option::Option::Some; + let mut state = serializer.serialize_map(std::option::Option::None)?; + if !self.serialized_record_batch.is_empty() { + struct __With<'a>(&'a ::bytes::Bytes); + impl<'a> serde::ser::Serialize for __With<'a> { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::ser::Serializer, + { + serde_with::As::::serialize(self.0, serializer) + } + } + state.serialize_entry( + "serializedRecordBatch", + &__With(&self.serialized_record_batch), + )?; + } + if !self._unknown_fields.is_empty() { + for (key, value) in self._unknown_fields.iter() { + state.serialize_entry(key, &value)?; + } + } + state.end() + } +} + +#[doc(hidden)] +impl serde::ser::Serialize for super::ArrowSerializationOptions { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::ser::Serializer, + { + use serde::ser::SerializeMap; + #[allow(unused_imports)] + use std::option::Option::Some; + let mut state = serializer.serialize_map(std::option::Option::None)?; + if !wkt::internal::is_default(&self.buffer_compression) { + state.serialize_entry("bufferCompression", &self.buffer_compression)?; + } + if !wkt::internal::is_default(&self.picos_timestamp_precision) { + state.serialize_entry("picosTimestampPrecision", &self.picos_timestamp_precision)?; + } + if !self._unknown_fields.is_empty() { + for (key, value) in self._unknown_fields.iter() { + state.serialize_entry(key, &value)?; + } + } + state.end() + } +} + #[doc(hidden)] impl serde::ser::Serialize for super::BigLakeConfiguration { fn serialize(&self, serializer: S) -> std::result::Result @@ -2282,6 +2372,12 @@ impl serde::ser::Serialize for super::QueryRequest { if !wkt::internal::is_default(&self.write_incremental_results) { state.serialize_entry("writeIncrementalResults", &self.write_incremental_results)?; } + if !wkt::internal::is_default(&self.query_results_format) { + state.serialize_entry("queryResultsFormat", &self.query_results_format)?; + } + if let Some(value) = self.arrow_serialization_options() { + state.serialize_entry("arrowSerializationOptions", value)?; + } if !self._unknown_fields.is_empty() { for (key, value) in self._unknown_fields.iter() { state.serialize_entry(key, &value)?; @@ -2452,6 +2548,27 @@ impl serde::ser::Serialize for super::QueryResponse { } state.serialize_entry("endTime", &__With(&self.end_time))?; } + if let Some(value) = self.arrow_schema() { + state.serialize_entry("arrowSchema", value)?; + } + if let Some(value) = self.arrow_record_batch() { + state.serialize_entry("arrowRecordBatch", value)?; + } + if !wkt::internal::is_default(&self.page_row_count) { + struct __With<'a>(&'a i64); + impl<'a> serde::ser::Serialize for __With<'a> { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::ser::Serializer, + { + serde_with::As::::serialize(self.0, serializer) + } + } + state.serialize_entry("pageRowCount", &__With(&self.page_row_count))?; + } + if !self.statement_type.is_empty() { + state.serialize_entry("statementType", &self.statement_type)?; + } if !self._unknown_fields.is_empty() { for (key, value) in self._unknown_fields.iter() { state.serialize_entry(key, &value)?; @@ -4504,6 +4621,62 @@ impl serde::ser::Serialize for super::JobStatistics2 { if self.gen_ai_stats.is_some() { state.serialize_entry("genAiStats", &self.gen_ai_stats)?; } + if !self.object_storage_stats.is_empty() { + state.serialize_entry("objectStorageStats", &self.object_storage_stats)?; + } + if !self._unknown_fields.is_empty() { + for (key, value) in self._unknown_fields.iter() { + state.serialize_entry(key, &value)?; + } + } + state.end() + } +} + +#[doc(hidden)] +impl serde::ser::Serialize for super::ObjectStorageStats { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::ser::Serializer, + { + use serde::ser::SerializeMap; + #[allow(unused_imports)] + use std::option::Option::Some; + let mut state = serializer.serialize_map(std::option::Option::None)?; + if self.cloud_provider.is_some() { + state.serialize_entry("cloudProvider", &self.cloud_provider)?; + } + if self.object_storage_bytes_read.is_some() { + struct __With<'a>(&'a std::option::Option); + impl<'a> serde::ser::Serialize for __With<'a> { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::ser::Serializer, + { + serde_with::As::>::serialize( + self.0, serializer, + ) + } + } + state.serialize_entry( + "objectStorageBytesRead", + &__With(&self.object_storage_bytes_read), + )?; + } + if self.cache_bytes_read.is_some() { + struct __With<'a>(&'a std::option::Option); + impl<'a> serde::ser::Serialize for __With<'a> { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::ser::Serializer, + { + serde_with::As::>::serialize( + self.0, serializer, + ) + } + } + state.serialize_entry("cacheBytesRead", &__With(&self.cache_bytes_read))?; + } if !self._unknown_fields.is_empty() { for (key, value) in self._unknown_fields.iter() { state.serialize_entry(key, &value)?; @@ -4696,6 +4869,9 @@ impl serde::ser::Serialize for super::CopyJobStatistics { } state.serialize_entry("copiedLogicalBytes", &__With(&self.copied_logical_bytes))?; } + if !self.remote_destination_region.is_empty() { + state.serialize_entry("remoteDestinationRegion", &self.remote_destination_region)?; + } if !self._unknown_fields.is_empty() { for (key, value) in self._unknown_fields.iter() { state.serialize_entry(key, &value)?; @@ -5050,6 +5226,15 @@ impl serde::ser::Serialize for super::JobStatistics { if !self.reservation_group_path.is_empty() { state.serialize_entry("reservationGroupPath", &self.reservation_group_path)?; } + if !self.global_query_remote_regions.is_empty() { + state.serialize_entry( + "globalQueryRemoteRegions", + &self.global_query_remote_regions, + )?; + } + if self.parent_global_query_job.is_some() { + state.serialize_entry("parentGlobalQueryJob", &self.parent_global_query_job)?; + } if !self._unknown_fields.is_empty() { for (key, value) in self._unknown_fields.iter() { state.serialize_entry(key, &value)?; @@ -5425,6 +5610,42 @@ impl serde::ser::Serialize for super::partition_skew::SkewSource { } state.serialize_entry("stageId", &__With(&self.stage_id))?; } + if !wkt::internal::is_default(&self.output_bytes_median) { + struct __With<'a>(&'a i64); + impl<'a> serde::ser::Serialize for __With<'a> { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::ser::Serializer, + { + serde_with::As::::serialize(self.0, serializer) + } + } + state.serialize_entry("outputBytesMedian", &__With(&self.output_bytes_median))?; + } + if !wkt::internal::is_default(&self.output_bytes_p95) { + struct __With<'a>(&'a i64); + impl<'a> serde::ser::Serialize for __With<'a> { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::ser::Serializer, + { + serde_with::As::::serialize(self.0, serializer) + } + } + state.serialize_entry("outputBytesP95", &__With(&self.output_bytes_p95))?; + } + if !wkt::internal::is_default(&self.output_bytes_max) { + struct __With<'a>(&'a i64); + impl<'a> serde::ser::Serialize for __With<'a> { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::ser::Serializer, + { + serde_with::As::::serialize(self.0, serializer) + } + } + state.serialize_entry("outputBytesMax", &__With(&self.output_bytes_max))?; + } if !self._unknown_fields.is_empty() { for (key, value) in self._unknown_fields.iter() { state.serialize_entry(key, &value)?; @@ -9863,6 +10084,9 @@ impl serde::ser::Serialize for super::routine::Argument { if self.data_type.is_some() { state.serialize_entry("dataType", &self.data_type)?; } + if self.table_type.is_some() { + state.serialize_entry("tableType", &self.table_type)?; + } if self.is_aggregate.is_some() { state.serialize_entry("isAggregate", &self.is_aggregate)?; } diff --git a/tests/bigquery/Cargo.toml b/tests/bigquery/Cargo.toml index da70597ad1..4a8b804f12 100644 --- a/tests/bigquery/Cargo.toml +++ b/tests/bigquery/Cargo.toml @@ -28,7 +28,7 @@ log-integration-tests = [] [dependencies] anyhow.workspace = true -arrow = { workspace = true, features = ["ipc"] } +arrow = { workspace = true, features = ["ipc", "ipc_compression"] } bytes.workspace = true futures.workspace = true google-cloud-auth = { workspace = true } From b70e4afdc8f393ad9be1ab3b91b2d2ee17b3d7af Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Fri, 21 Aug 2026 19:22:12 +0000 Subject: [PATCH 2/3] impl: add benchmark and unstable flag --- Cargo.lock | 11 + Cargo.toml | 2 + .../benchmarks/arrow_jobs_query/Cargo.toml | 32 ++ .../benchmarks/arrow_jobs_query/src/main.rs | 409 ++++++++++++++++++ src/bigquery/src/query/execution.rs | 21 +- 5 files changed, 465 insertions(+), 10 deletions(-) create mode 100644 src/bigquery/benchmarks/arrow_jobs_query/Cargo.toml create mode 100644 src/bigquery/benchmarks/arrow_jobs_query/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 2934b0ad79..2f59080799 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -396,6 +396,17 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" +[[package]] +name = "bigquery-benchmark-arrow-jobs-query" +version = "0.0.0" +dependencies = [ + "anyhow", + "clap", + "google-cloud-bigquery", + "google-cloud-wkt", + "tokio", +] + [[package]] name = "bigquery-samples" version = "0.0.0" diff --git a/Cargo.toml b/Cargo.toml index 1b6f0d9a0a..ecbe8a68ce 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,6 +68,7 @@ members = [ "guide/samples", "src/auth", "src/bigquery", + "src/bigquery/benchmarks/arrow_jobs_query", "src/bigquery-derive", "src/bigquery-write", "src/bigquery-write/grpc-mock", @@ -609,6 +610,7 @@ unexpected_cfgs = { level = "deny", check-cfg = [ 'cfg(google_cloud_unstable_storage_bidi)', 'cfg(google_cloud_unstable_tracing)', 'cfg(google_cloud_unstable_trust_boundaries)', + 'cfg(google_cloud_unstable_bigquery_arrow)', # Enables re-generation of protos. Only needed on major version updates to # Prost and/or Tonic. 'cfg(google_cloud_generate_protos)', diff --git a/src/bigquery/benchmarks/arrow_jobs_query/Cargo.toml b/src/bigquery/benchmarks/arrow_jobs_query/Cargo.toml new file mode 100644 index 0000000000..2ce209a1de --- /dev/null +++ b/src/bigquery/benchmarks/arrow_jobs_query/Cargo.toml @@ -0,0 +1,32 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[package] +name = "bigquery-benchmark-arrow-jobs-query" +version = "0.0.0" +publish = false +edition.workspace = true +authors.workspace = true +license.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +anyhow.workspace = true +clap = { workspace = true, features = ["derive", "env", "help", "std"] } +google-cloud-bigquery = { workspace = true, features = ["default"] } +tokio = { workspace = true, features = ["full"] } +wkt.workspace = true diff --git a/src/bigquery/benchmarks/arrow_jobs_query/src/main.rs b/src/bigquery/benchmarks/arrow_jobs_query/src/main.rs new file mode 100644 index 0000000000..dc0a20a906 --- /dev/null +++ b/src/bigquery/benchmarks/arrow_jobs_query/src/main.rs @@ -0,0 +1,409 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use clap::{Parser, ValueEnum}; +use google_cloud_bigquery::FromRow; +use google_cloud_bigquery::client::BigQuery; +use std::time::{Duration, Instant}; + +#[derive(Parser, Debug)] +#[command( + name = "bigquery-benchmark-arrow-jobs-query", + about = "BigQuery jobs.query Benchmark: Arrow results format vs standard JSON" +)] +struct Args { + /// GCP Project ID (reads from GOOGLE_CLOUD_PROJECT if omitted). + #[arg(long, env = "GOOGLE_CLOUD_PROJECT")] + project_id: Option, + + /// Scenario to execute. + #[arg(long, value_enum, default_value = "synthetic-100k")] + scenario: Scenario, + + /// Custom query to run when scenario is `custom`. + #[arg(long)] + query: Option, + + /// Number of benchmark measurement iterations. + #[arg(long, default_value_t = 5)] + iterations: usize, + + /// Number of warmup iterations before measuring. + #[arg(long, default_value_t = 1)] + warmup: usize, + + /// Whether to enable BigQuery server-side query cache (default: false). + #[arg(long, default_value_t = false)] + use_query_cache: bool, + + /// Whether to deserialize rows into typed structs using FromRow. + #[arg(long, default_value_t = true)] + typed: bool, +} + +#[derive(ValueEnum, Clone, Copy, Debug, PartialEq, Eq)] +enum Scenario { + #[value(name = "synthetic-1")] + Synthetic1, + #[value(name = "synthetic-100")] + Synthetic100, + #[value(name = "synthetic-1k")] + Synthetic1k, + #[value(name = "synthetic-10k")] + Synthetic10k, + #[value(name = "synthetic-50k")] + Synthetic50k, + #[value(name = "synthetic-100k")] + Synthetic100k, + #[value(name = "synthetic-500k")] + Synthetic500k, + #[value(name = "wikipedia-10k")] + Wikipedia10k, + #[value(name = "wikipedia-100k")] + Wikipedia100k, + #[value(name = "custom")] + Custom, +} + +impl Scenario { + fn query(&self, custom_query: Option<&str>) -> String { + match self { + Scenario::Synthetic1 => Self::synthetic_query(1), + Scenario::Synthetic100 => Self::synthetic_query(100), + Scenario::Synthetic1k => Self::synthetic_query(1_000), + Scenario::Synthetic10k => Self::synthetic_query(10_000), + Scenario::Synthetic50k => Self::synthetic_query(50_000), + Scenario::Synthetic100k => Self::synthetic_query(100_000), + Scenario::Synthetic500k => Self::synthetic_query(500_000), + Scenario::Wikipedia10k => Self::wikipedia_query(10_000), + Scenario::Wikipedia100k => Self::wikipedia_query(100_000), + Scenario::Custom => custom_query + .expect("custom query must be provided when scenario is 'custom'") + .to_string(), + } + } + + fn synthetic_query(row_count: usize) -> String { + format!( + "SELECT \ + x AS id, \ + CONCAT('row_item_name_', CAST(x AS STRING)) AS name, \ + CAST(x AS FLOAT64) * 1.25 AS score, \ + (MOD(x, 2) = 0) AS is_even, \ + CURRENT_TIMESTAMP() AS created_at \ + FROM UNNEST(GENERATE_ARRAY(1, {row_count})) AS x" + ) + } + + fn wikipedia_query(limit: usize) -> String { + format!( + "SELECT \ + title, \ + id, \ + language, \ + wp_namespace, \ + is_redirect, \ + revision_id, \ + timestamp, \ + contributor_ip, \ + contributor_id, \ + contributor_username, \ + comment, \ + num_characters \ + FROM `bigquery-public-data.samples.wikipedia` \ + LIMIT {limit}" + ) + } + + fn is_wikipedia(&self) -> bool { + matches!(self, Scenario::Wikipedia10k | Scenario::Wikipedia100k) + } +} + +#[derive(FromRow, Debug, PartialEq)] +#[allow(dead_code)] +struct SyntheticRow { + id: i64, + name: String, + score: f64, + is_even: bool, + created_at: wkt::Timestamp, +} + +#[derive(FromRow, Debug, PartialEq)] +#[allow(dead_code)] +struct WikipediaRow { + title: Option, + id: Option, + language: Option, + wp_namespace: Option, + is_redirect: Option, + revision_id: Option, + timestamp: Option, + contributor_ip: Option, + contributor_id: Option, + contributor_username: Option, + comment: Option, + num_characters: Option, +} + +struct IterationResult { + query_duration: Duration, + read_duration: Duration, + total_duration: Duration, + rows_count: usize, +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let args = Args::parse(); + + let project_id = args.project_id.ok_or_else(|| { + anyhow::anyhow!( + "Project ID must be provided via --project-id or GOOGLE_CLOUD_PROJECT env var" + ) + })?; + + let sql_query = args.scenario.query(args.query.as_deref()); + + println!("================================================================================"); + println!(" BigQuery Query Benchmark"); + println!("================================================================================"); + #[cfg(google_cloud_unstable_bigquery_arrow)] + println!(" Arrow Acceleration: ENABLED (--cfg google_cloud_unstable_bigquery_arrow)"); + #[cfg(not(google_cloud_unstable_bigquery_arrow))] + println!(" Arrow Acceleration: DISABLED (Standard JSON mode)"); + println!(" Project ID: {project_id}"); + println!(" Scenario: {:?}", args.scenario); + println!(" Warmup Iterations: {}", args.warmup); + println!(" Measured Runs: {}", args.iterations); + println!(" Use Query Cache: {}", args.use_query_cache); + println!(" Typed Deserialization: {}", args.typed); + println!("================================================================================"); + println!(); + + let client = BigQuery::builder() + .with_project_id(&project_id) + .build() + .await?; + + // Warmup runs + if args.warmup > 0 { + println!("Running {} warmup iteration(s)...", args.warmup); + for i in 1..=args.warmup { + print!(" Warmup {i}/{}: ", args.warmup); + let result = run_single_query( + &client, + &project_id, + &sql_query, + args.scenario, + args.use_query_cache, + args.typed, + ) + .await?; + println!( + "done ({} rows, query: {:.2?}, read: {:.2?}, total: {:.2?})", + result.rows_count, + result.query_duration, + result.read_duration, + result.total_duration + ); + } + println!(); + } + + // Benchmark measured runs + println!("Running {} benchmark measurement(s)...", args.iterations); + println!("--------------------------------------------------------------------------------"); + println!( + "{:<6} | {:<12} | {:<12} | {:<12} | {:<10} | {:<14}", + "Run", "Query Time", "Read/Iter", "Total Time", "Rows", "Throughput" + ); + println!("--------------------------------------------------------------------------------"); + + let mut results = Vec::with_capacity(args.iterations); + for i in 1..=args.iterations { + let result = run_single_query( + &client, + &project_id, + &sql_query, + args.scenario, + args.use_query_cache, + args.typed, + ) + .await?; + let rps = if result.read_duration.as_secs_f64() > 0.0 { + result.rows_count as f64 / result.read_duration.as_secs_f64() + } else { + 0.0 + }; + + println!( + "{:<6} | {:<12.2?} | {:<12.2?} | {:<12.2?} | {:<10} | {:>10.0} rows/s", + format!("#{i}"), + result.query_duration, + result.read_duration, + result.total_duration, + result.rows_count, + rps + ); + results.push(result); + } + println!("--------------------------------------------------------------------------------"); + + // Print summary statistics + print_summary(&results); + + println!(); + println!("Tip: Compare Arrow vs JSON by running:"); + println!( + " Arrow: RUSTFLAGS=\"--cfg google_cloud_unstable_bigquery_arrow\" cargo run --release -p bigquery-benchmark-arrow-jobs-query -- --scenario {:?}", + args.scenario + ); + println!( + " JSON: cargo run --release -p bigquery-benchmark-arrow-jobs-query -- --scenario {:?}", + args.scenario + ); + println!(); + + Ok(()) +} + +async fn run_single_query( + client: &BigQuery, + project_id: &str, + query_str: &str, + scenario: Scenario, + use_query_cache: bool, + typed: bool, +) -> anyhow::Result { + let start_total = Instant::now(); + + // 1. Submit query and wait until complete + let start_query = Instant::now(); + let complete_query = client + .query(query_str) + .with_project_id(project_id) + .set_use_query_cache(use_query_cache) + .until_done() + .await?; + let query_duration = start_query.elapsed(); + + // 2. Read and deserialize rows + let start_read = Instant::now(); + let mut iter = complete_query.read(); + let mut rows_count = 0; + + if typed { + if scenario.is_wikipedia() { + while let Some(row_res) = iter.next().await { + let row = row_res?; + let _typed_row: WikipediaRow = row.try_into()?; + rows_count += 1; + } + } else { + while let Some(row_res) = iter.next().await { + let row = row_res?; + let _typed_row: SyntheticRow = row.try_into()?; + rows_count += 1; + } + } + } else { + while let Some(row_res) = iter.next().await { + let _row = row_res?; + rows_count += 1; + } + } + let read_duration = start_read.elapsed(); + let total_duration = start_total.elapsed(); + + Ok(IterationResult { + query_duration, + read_duration, + total_duration, + rows_count, + }) +} + +fn print_summary(results: &[IterationResult]) { + if results.is_empty() { + return; + } + + let n = results.len() as f64; + let query_times: Vec = results + .iter() + .map(|r| r.query_duration.as_secs_f64()) + .collect(); + let read_times: Vec = results + .iter() + .map(|r| r.read_duration.as_secs_f64()) + .collect(); + let total_times: Vec = results + .iter() + .map(|r| r.total_duration.as_secs_f64()) + .collect(); + let throughputs: Vec = results + .iter() + .map(|r| { + if r.read_duration.as_secs_f64() > 0.0 { + r.rows_count as f64 / r.read_duration.as_secs_f64() + } else { + 0.0 + } + }) + .collect(); + + let avg = |v: &[f64]| v.iter().sum::() / n; + let min = |v: &[f64]| v.iter().cloned().fold(f64::INFINITY, f64::min); + let max = |v: &[f64]| v.iter().cloned().fold(f64::NEG_INFINITY, f64::max); + let std_dev = |v: &[f64], mean: f64| { + let variance = v.iter().map(|x| (x - mean).powi(2)).sum::() / n; + variance.sqrt() + }; + + let q_avg = avg(&query_times); + let r_avg = avg(&read_times); + let t_avg = avg(&total_times); + let tp_avg = avg(&throughputs); + + println!("Summary Statistics (over {} runs):", results.len()); + println!( + " Query Execution Time: avg: {:.2?} (min: {:.2?}, max: {:.2?}, stddev: {:.2?})", + Duration::from_secs_f64(q_avg), + Duration::from_secs_f64(min(&query_times)), + Duration::from_secs_f64(max(&query_times)), + Duration::from_secs_f64(std_dev(&query_times, q_avg)) + ); + println!( + " Row Reading & Parsing: avg: {:.2?} (min: {:.2?}, max: {:.2?}, stddev: {:.2?})", + Duration::from_secs_f64(r_avg), + Duration::from_secs_f64(min(&read_times)), + Duration::from_secs_f64(max(&read_times)), + Duration::from_secs_f64(std_dev(&read_times, r_avg)) + ); + println!( + " Total End-to-End Time: avg: {:.2?} (min: {:.2?}, max: {:.2?}, stddev: {:.2?})", + Duration::from_secs_f64(t_avg), + Duration::from_secs_f64(min(&total_times)), + Duration::from_secs_f64(max(&total_times)), + Duration::from_secs_f64(std_dev(&total_times, t_avg)) + ); + println!( + " Row Throughput: avg: {:.0} rows/s (min: {:.0}, max: {:.0})", + tp_avg, + min(&throughputs), + max(&throughputs) + ); +} diff --git a/src/bigquery/src/query/execution.rs b/src/bigquery/src/query/execution.rs index a9f8e04fa6..31dbc62367 100644 --- a/src/bigquery/src/query/execution.rs +++ b/src/bigquery/src/query/execution.rs @@ -190,21 +190,22 @@ impl RetryContext { let query_request_id = generate_prefixed_id(QUERY_REQUEST_ID_PREFIX); let query_request: QueryRequest = self.template.request.clone().into(); + let query_request = query_request.set_format_options( + crate::model::DataFormatOptions::new().set_use_int64_timestamp(true), + ); + #[cfg(google_cloud_unstable_bigquery_arrow)] let query_request = query_request - .set_format_options( - crate::model::DataFormatOptions::new() - .set_use_int64_timestamp(true), - ) .set_query_results_format(crate::model::query_request::QueryResultsFormat::Arrow) .set_results_format_serialization_options( crate::model::query_request::ResultsFormatSerializationOptions::ArrowSerializationOptions( - Box::new(crate::model::ArrowSerializationOptions::new() - .set_buffer_compression( + Box::new( + crate::model::ArrowSerializationOptions::new().set_buffer_compression( crate::model::arrow_serialization_options::CompressionCodec::Zstd, - )) - ) - ) - .set_request_id(query_request_id); + ), + ), + ), + ); + let query_request = query_request.set_request_id(query_request_id); let req = PostQueryRequest::new() .set_project_id(project_id) .set_query_request(query_request); From 86ee76ca2d290ece2a6ec2afce6afc6efcc4066a Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Fri, 21 Aug 2026 20:55:21 +0000 Subject: [PATCH 3/3] impl: track memory on benchmark --- Cargo.lock | 7 ++ .../benchmarks/arrow_jobs_query/Cargo.toml | 1 + .../benchmarks/arrow_jobs_query/src/main.rs | 119 ++++++++++++++---- 3 files changed, 106 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2f59080799..fd3c1e4186 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -404,6 +404,7 @@ dependencies = [ "clap", "google-cloud-bigquery", "google-cloud-wkt", + "stats_alloc", "tokio", ] @@ -9352,6 +9353,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "stats_alloc" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c0e04424e733e69714ca1bbb9204c1a57f09f5493439520f9f68c132ad25eec" + [[package]] name = "storage-benchmark-appendable-object" version = "0.0.0" diff --git a/src/bigquery/benchmarks/arrow_jobs_query/Cargo.toml b/src/bigquery/benchmarks/arrow_jobs_query/Cargo.toml index 2ce209a1de..bb6838335d 100644 --- a/src/bigquery/benchmarks/arrow_jobs_query/Cargo.toml +++ b/src/bigquery/benchmarks/arrow_jobs_query/Cargo.toml @@ -30,3 +30,4 @@ clap = { workspace = true, features = ["derive", "env", "help", google-cloud-bigquery = { workspace = true, features = ["default"] } tokio = { workspace = true, features = ["full"] } wkt.workspace = true +stats_alloc = "0.1" diff --git a/src/bigquery/benchmarks/arrow_jobs_query/src/main.rs b/src/bigquery/benchmarks/arrow_jobs_query/src/main.rs index dc0a20a906..94c9e82a17 100644 --- a/src/bigquery/benchmarks/arrow_jobs_query/src/main.rs +++ b/src/bigquery/benchmarks/arrow_jobs_query/src/main.rs @@ -15,12 +15,33 @@ use clap::{Parser, ValueEnum}; use google_cloud_bigquery::FromRow; use google_cloud_bigquery::client::BigQuery; +use stats_alloc::{Region, StatsAlloc}; +use std::alloc::System; use std::time::{Duration, Instant}; +#[global_allocator] +static GLOBAL: StatsAlloc = StatsAlloc::system(); + +fn format_bytes(bytes: usize) -> String { + if bytes >= 1024 * 1024 * 1024 { + format!("{:.2} GB", bytes as f64 / (1024.0 * 1024.0 * 1024.0)) + } else if bytes >= 1024 * 1024 { + format!("{:.2} MB", bytes as f64 / (1024.0 * 1024.0)) + } else if bytes >= 1024 { + format!("{:.2} KB", bytes as f64 / 1024.0) + } else { + format!("{bytes} B") + } +} + +// ============================================================================ +// CLI Arguments and Scenarios +// ============================================================================ + #[derive(Parser, Debug)] #[command( name = "bigquery-benchmark-arrow-jobs-query", - about = "BigQuery jobs.query Benchmark: Arrow results format vs standard JSON" + about = "BigQuery jobs.query Benchmark: Arrow results format vs standard JSON (Speed & Allocations)" )] struct Args { /// GCP Project ID (reads from GOOGLE_CLOUD_PROJECT if omitted). @@ -163,6 +184,8 @@ struct IterationResult { read_duration: Duration, total_duration: Duration, rows_count: usize, + bytes_allocated: usize, + allocations_count: usize, } #[tokio::main] @@ -177,20 +200,26 @@ async fn main() -> anyhow::Result<()> { let sql_query = args.scenario.query(args.query.as_deref()); - println!("================================================================================"); - println!(" BigQuery Query Benchmark"); - println!("================================================================================"); + println!( + "==========================================================================================================" + ); + println!(" BigQuery Query Benchmark (Speed & Memory)"); + println!( + "==========================================================================================================" + ); #[cfg(google_cloud_unstable_bigquery_arrow)] - println!(" Arrow Acceleration: ENABLED (--cfg google_cloud_unstable_bigquery_arrow)"); + println!(" Arrow Acceleration: ENABLED (--cfg google_cloud_unstable_bigquery_arrow)"); #[cfg(not(google_cloud_unstable_bigquery_arrow))] - println!(" Arrow Acceleration: DISABLED (Standard JSON mode)"); - println!(" Project ID: {project_id}"); - println!(" Scenario: {:?}", args.scenario); - println!(" Warmup Iterations: {}", args.warmup); - println!(" Measured Runs: {}", args.iterations); - println!(" Use Query Cache: {}", args.use_query_cache); + println!(" Arrow Acceleration: DISABLED (Standard JSON mode)"); + println!(" Project ID: {project_id}"); + println!(" Scenario: {:?}", args.scenario); + println!(" Warmup Iterations: {}", args.warmup); + println!(" Measured Runs: {}", args.iterations); + println!(" Use Query Cache: {}", args.use_query_cache); println!(" Typed Deserialization: {}", args.typed); - println!("================================================================================"); + println!( + "==========================================================================================================" + ); println!(); let client = BigQuery::builder() @@ -213,11 +242,12 @@ async fn main() -> anyhow::Result<()> { ) .await?; println!( - "done ({} rows, query: {:.2?}, read: {:.2?}, total: {:.2?})", + "done ({} rows, query: {:.2?}, read: {:.2?}, total: {:.2?}, allocated: {})", result.rows_count, result.query_duration, result.read_duration, - result.total_duration + result.total_duration, + format_bytes(result.bytes_allocated) ); } println!(); @@ -225,12 +255,23 @@ async fn main() -> anyhow::Result<()> { // Benchmark measured runs println!("Running {} benchmark measurement(s)...", args.iterations); - println!("--------------------------------------------------------------------------------"); println!( - "{:<6} | {:<12} | {:<12} | {:<12} | {:<10} | {:<14}", - "Run", "Query Time", "Read/Iter", "Total Time", "Rows", "Throughput" + "------------------------------------------------------------------------------------------------------------------" + ); + println!( + "{:<6} | {:<12} | {:<12} | {:<12} | {:<10} | {:<14} | {:<11} | {:<12}", + "Run", + "Query Time", + "Read/Iter", + "Total Time", + "Rows", + "Throughput", + "Allocated", + "Allocations" + ); + println!( + "------------------------------------------------------------------------------------------------------------------" ); - println!("--------------------------------------------------------------------------------"); let mut results = Vec::with_capacity(args.iterations); for i in 1..=args.iterations { @@ -250,17 +291,21 @@ async fn main() -> anyhow::Result<()> { }; println!( - "{:<6} | {:<12.2?} | {:<12.2?} | {:<12.2?} | {:<10} | {:>10.0} rows/s", + "{:<6} | {:<12.2?} | {:<12.2?} | {:<12.2?} | {:<10} | {:>10.0} rows/s | {:<11} | {:>12}", format!("#{i}"), result.query_duration, result.read_duration, result.total_duration, result.rows_count, - rps + rps, + format_bytes(result.bytes_allocated), + result.allocations_count ); results.push(result); } - println!("--------------------------------------------------------------------------------"); + println!( + "------------------------------------------------------------------------------------------------------------------" + ); // Print summary statistics print_summary(&results); @@ -288,6 +333,7 @@ async fn run_single_query( use_query_cache: bool, typed: bool, ) -> anyhow::Result { + let region = Region::new(&GLOBAL); let start_total = Instant::now(); // 1. Submit query and wait until complete @@ -328,11 +374,17 @@ async fn run_single_query( let read_duration = start_read.elapsed(); let total_duration = start_total.elapsed(); + let stats = region.change(); + let bytes_allocated = stats.bytes_allocated; + let allocations_count = stats.allocations; + Ok(IterationResult { query_duration, read_duration, total_duration, rows_count, + bytes_allocated, + allocations_count, }) } @@ -364,6 +416,8 @@ fn print_summary(results: &[IterationResult]) { } }) .collect(); + let bytes_allocs: Vec = results.iter().map(|r| r.bytes_allocated as f64).collect(); + let alloc_counts: Vec = results.iter().map(|r| r.allocations_count as f64).collect(); let avg = |v: &[f64]| v.iter().sum::() / n; let min = |v: &[f64]| v.iter().cloned().fold(f64::INFINITY, f64::min); @@ -377,6 +431,15 @@ fn print_summary(results: &[IterationResult]) { let r_avg = avg(&read_times); let t_avg = avg(&total_times); let tp_avg = avg(&throughputs); + let bytes_avg = avg(&bytes_allocs); + let count_avg = avg(&alloc_counts); + + let rows_avg = results[0].rows_count as f64; + let bytes_per_row = if rows_avg > 0.0 { + bytes_avg / rows_avg + } else { + 0.0 + }; println!("Summary Statistics (over {} runs):", results.len()); println!( @@ -406,4 +469,18 @@ fn print_summary(results: &[IterationResult]) { min(&throughputs), max(&throughputs) ); + println!( + " Total Heap Allocated: avg: {} ({:.1} bytes/row)", + format_bytes(bytes_avg as usize), + bytes_per_row + ); + println!( + " Total Allocations: avg: {:.0} allocs ({:.1} allocs/row)", + count_avg, + if rows_avg > 0.0 { + count_avg / rows_avg + } else { + 0.0 + } + ); }