feat: added support for per type and partition export for Cloud Asset API (#398)

Clients can now specify two more args when export assets to bigquery

PiperOrigin-RevId: 331912851

Source-Author: Google APIs <noreply@google.com>
Source-Date: Tue Sep 15 20:04:02 2020 -0700
Source-Repo: googleapis/googleapis
Source-Sha: 5e53d6b6dde0e72fa9510ec1d796176d128afa40
Source-Link: googleapis/googleapis@5e53d6b

Author: yoshi-automation

packages/google-cloud-asset/protos/google/cloud/asset/v1/asset_service.proto

@@ -393,6 +393,83 @@ message BigQueryDestination {
   // is `FALSE` or unset and the destination table already exists, the export
   // call returns an INVALID_ARGUMEMT error.
   bool force = 3;
+
+  // [partition_spec] determines whether to export to partitioned table(s) and
+  // how to partition the data.
+  //
+  // If [partition_spec] is unset or [partition_spec.partion_key] is unset or
+  // `PARTITION_KEY_UNSPECIFIED`, the snapshot results will be exported to
+  // non-partitioned table(s). [force] will decide whether to overwrite existing
+  // table(s).
+  //
+  // If [partition_spec] is specified. First, the snapshot results will be
+  // written to partitioned table(s) with two additional timestamp columns,
+  // readTime and requestTime, one of which will be the partition key. Secondly,
+  // in the case when any destination table already exists, it will first try to
+  // update existing table's schema as necessary by appending additional
+  // columns. Then, if [force] is `TRUE`, the corresponding partition will be
+  // overwritten by the snapshot results (data in different partitions will
+  // remain intact); if [force] is unset or `FALSE`, it will append the data. An
+  // error will be returned if the schema update or data appension fails.
+  PartitionSpec partition_spec = 4;
+
+  // If this flag is `TRUE`, the snapshot results will be written to one or
+  // multiple tables, each of which contains results of one asset type. The
+  // [force] and [partition_spec] fields will apply to each of them.
+  //
+  // Field [table] will be concatenated with "_" and the asset type names (see
+  // https://cloud.google.com/asset-inventory/docs/supported-asset-types for
+  // supported asset types) to construct per-asset-type table names, in which
+  // all non-alphanumeric characters like "." and "/" will be substituted by
+  // "_". Example: if field [table] is "mytable" and snapshot results
+  // contain "storage.googleapis.com/Bucket" assets, the corresponding table
+  // name will be "mytable_storage_googleapis_com_Bucket". If any of these
+  // tables does not exist, a new table with the concatenated name will be
+  // created.
+  //
+  // When [content_type] in the ExportAssetsRequest is `RESOURCE`, the schema of
+  // each table will include RECORD-type columns mapped to the nested fields in
+  // the Asset.resource.data field of that asset type (up to the 15 nested level
+  // BigQuery supports
+  // (https://cloud.google.com/bigquery/docs/nested-repeated#limitations)). The
+  // fields in >15 nested levels will be stored in JSON format string as a child
+  // column of its parent RECORD column.
+  //
+  // If error occurs when exporting to any table, the whole export call will
+  // return an error but the export results that already succeed will persist.
+  // Example: if exporting to table_type_A succeeds when exporting to
+  // table_type_B fails during one export call, the results in table_type_A will
+  // persist and there will not be partial results persisting in a table.
+  bool separate_tables_per_asset_type = 5;
+}
+
+// Specifications of BigQuery partitioned table as export destination.
+message PartitionSpec {
+  // This enum is used to determine the partition key column when exporting
+  // assets to BigQuery partitioned table(s). Note that, if the partition key is
+  // a timestamp column, the actual partition is based on its date value
+  // (expressed in UTC. see details in
+  // https://cloud.google.com/bigquery/docs/partitioned-tables#date_timestamp_partitioned_tables).
+  enum PartitionKey {
+    // Unspecified partition key. If used, it means using non-partitioned table.
+    PARTITION_KEY_UNSPECIFIED = 0;
+
+    // The time when the snapshot is taken. If specified as partition key, the
+    // result table(s) is partitoned by the additional timestamp column,
+    // readTime. If [read_time] in ExportAssetsRequest is specified, the
+    // readTime column's value will be the same as it. Otherwise, its value will
+    // be the current time that is used to take the snapshot.
+    READ_TIME = 1;
+
+    // The time when the request is received and started to be processed. If
+    // specified as partition key, the result table(s) is partitoned by the
+    // requestTime column, an additional timestamp column representing when the
+    // request was received.
+    REQUEST_TIME = 2;
+  }
+
+  // The partition key for BigQuery partitioned table.
+  PartitionKey partition_key = 1;
 }
 
 // A Pub/Sub destination.