From 80a308fcca88f4c3778311b59d78273b44dceb12 Mon Sep 17 00:00:00 2001 From: sachitjaggi156 Date: Thu, 8 Jun 2023 06:49:56 +0000 Subject: [PATCH 01/19] Terraform Support for Dataplex Tasks. --- mmv1/products/dataplex/Task.yaml | 469 ++++++++++++++++++ mmv1/products/dataplex/product.yaml | 2 +- .../examples/dataplex_task_primary.tf.erb | 27 + 3 files changed, 497 insertions(+), 1 deletion(-) create mode 100644 mmv1/products/dataplex/Task.yaml create mode 100644 mmv1/templates/terraform/examples/dataplex_task_primary.tf.erb diff --git a/mmv1/products/dataplex/Task.yaml b/mmv1/products/dataplex/Task.yaml new file mode 100644 index 000000000000..c6d2edac48df --- /dev/null +++ b/mmv1/products/dataplex/Task.yaml @@ -0,0 +1,469 @@ +# Copyright 2023 Google Inc. +# Licensed under the Apache License, Version 2.0 (the License); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- !ruby/object:Api::Resource +name: 'Task' +base_url: 'projects/{{project}}/locations/{{location}}/lakes/{{lake}}/tasks/{{task_id}}' +self_link: 'projects/{{project}}/locations/{{location}}/lakes/{{lake}}/tasks/{{task_id}}' +create_url: 'projects/{{project}}/locations/{{location}}/lakes/{{lake}}/tasks?task_id={{task_id}}' +update_url: 'projects/{{project}}/locations/{{location}}/lakes/{{lake}}/tasks/{{task_id}}' +delete_url: 'projects/{{project}}/locations/{{location}}/lakes/{{lake}}/tasks/{{task_id}}' +description: | + Used to create Dataplex Tasks +async: !ruby/object:Provider::Terraform::PollAsync + check_response_func_existence: PollCheckForExistence + actions: ['create'] +references: !ruby/object:Api::Resource::ReferenceLinks + guides: + 'Official Documentation': 'https://cloud.google.com/dataplex/docs' + api: 'https://cloud.google.com/dataplex/docs/reference/rest' +import_format: ['projects/{{project}}/locations/{{location}}/lakes/{{lake}}/tasks/{{task_id}}'] +iam_policy: !ruby/object:Api::Resource::IamPolicy + exclude: false + method_name_separator: ':' + parent_resource_attribute: 'task' + fetch_iam_policy_verb: :POST + import_format: + [ + 'projects/{{project}}/locations/{{location}}/lakes/{{lake}}/tasks/{{task_id}}', + '{{task_id}}', + ] +parameters: + - !ruby/object:Api::Type::String + name: 'location' + url_param_only: true + immutable: true + description: | + The location in which the task will be created in. + - !ruby/object:Api::Type::String + name: 'lake' + url_param_only: true + immutable: true + description: | + The lake in which the task will be created in. + - !ruby/object:Api::Type::String + name: 'taskId' + url_param_only: true + immutable: true + description: | + The task Id of the task. +properties: + - !ruby/object:Api::Type::String + name: 'name' + output: true + description: | + The relative resource name of the task, of the form: projects/{project_number}/locations/{locationId}/lakes/{lakeId}/ tasks/{name}. + - !ruby/object:Api::Type::String + name: 'uid' + output: true + description: | + System generated globally unique ID for the task. This ID will be different if the task is deleted and re-created with the same name. + - !ruby/object:Api::Type::Time + name: 'createTime' + output: true + description: | + The time when the task was created. + - !ruby/object:Api::Type::Time + name: 'updateTime' + output: true + description: | + The time when the task was last updated. + - !ruby/object:Api::Type::String + name: 'description' + description: | + User-provided description of the task. + - !ruby/object:Api::Type::String + name: 'displayName' + description: | + User friendly display name. + - !ruby/object:Api::Type::Enum + name: 'state' + output: true + description: | + Current state of the task. + values: + - :STATE_UNSPECIFIED + - :ACTIVE + - :CREATING + - :DELETING + - :ACTION_REQUIRED + - !ruby/object:Api::Type::KeyValuePairs + name: 'labels' + description: | + User-defined labels for the task. + - !ruby/object:Api::Type::NestedObject + name: 'triggerSpec' + required: true + description: | + Configuration for the cluster + properties: + - !ruby/object:Api::Type::Enum + name: 'type' + required: true + immutable: true + description: | + Trigger type of the user-specified Task + values: + - :TYPE_UNSPECIFIED + - :ON_DEMAND + - :RECURRING + - !ruby/object:Api::Type::Time + name: 'startTime' + description: | + The first run of the task will be after this time. If not specified, the task will run shortly after being submitted if ON_DEMAND and based on the schedule if RECURRING. + - !ruby/object:Api::Type::Boolean + name: 'disabled' + description: | + Prevent the task from executing. This does not cancel already running tasks. It is intended to temporarily disable RECURRING tasks. + - !ruby/object:Api::Type::Integer + name: 'maxRetries' + description: | + Number of retry attempts before aborting. Set to zero to never attempt to retry a failed task. + - !ruby/object:Api::Type::String + name: 'schedule' + description: | + Cron schedule (https://en.wikipedia.org/wiki/Cron) for running tasks periodically. To explicitly set a timezone to the cron tab, apply a prefix in the cron tab: 'CRON_TZ=${IANA_TIME_ZONE}' or 'TZ=${IANA_TIME_ZONE}'. The ${IANA_TIME_ZONE} may only be a valid string from IANA time zone database. For example, CRON_TZ=America/New_York 1 * * * *, or TZ=America/New_York 1 * * * *. This field is required for RECURRING tasks. + - !ruby/object:Api::Type::NestedObject + name: 'executionSpec' + required: true + description: | + Configuration for the cluster + properties: + - !ruby/object:Api::Type::KeyValuePairs + name: 'args' + description: | + The arguments to pass to the task. The args can use placeholders of the format ${placeholder} as part of key/value string. These will be interpolated before passing the args to the driver. Currently supported placeholders: - ${taskId} - ${job_time} To pass positional args, set the key as TASK_ARGS. The value should be a comma-separated string of all the positional arguments. To use a delimiter other than comma, refer to https://cloud.google.com/sdk/gcloud/reference/topic/escaping. In case of other keys being present in the args, then TASK_ARGS will be passed as the last argument. An object containing a list of 'key': value pairs. Example: { 'name': 'wrench', 'mass': '1.3kg', 'count': '3' }. + - !ruby/object:Api::Type::String + name: 'serviceAccount' + required: true + description: | + Service account to use to execute a task. If not provided, the default Compute service account for the project is used. + - !ruby/object:Api::Type::String + name: 'project' + description: | + The project in which jobs are run. By default, the project containing the Lake is used. If a project is provided, the ExecutionSpec.service_account must belong to this project. + - !ruby/object:Api::Type::String + name: 'maxJobExecutionLifetime' + description: | + The maximum duration after which the job execution is expired. A duration in seconds with up to nine fractional digits, ending with 's'. Example: '3.5s'. + - !ruby/object:Api::Type::String + name: 'kmsKey' + description: | + The Cloud KMS key to use for encryption, of the form: projects/{project_number}/locations/{locationId}/keyRings/{key-ring-name}/cryptoKeys/{key-name}. + - !ruby/object:Api::Type::NestedObject + name: 'executionStatus' + output: true + description: | + Configuration for the cluster + properties: + - !ruby/object:Api::Type::String + name: 'updateTime' + output: true + description: | + Last update time of the status. + - !ruby/object:Api::Type::NestedObject + name: 'latestJob' + output: true + description: | + latest job execution. + properties: + - !ruby/object:Api::Type::String + name: 'name' + output: true + description: | + The relative resource name of the job, of the form: projects/{project_number}/locations/{locationId}/lakes/{lakeId}/tasks/{taskId}/jobs/{jobId}. + - !ruby/object:Api::Type::String + name: 'uid' + output: true + description: | + System generated globally unique ID for the job. + - !ruby/object:Api::Type::Time + name: 'startTime' + output: true + description: | + The time when the job was started. + - !ruby/object:Api::Type::Time + name: 'endTime' + output: true + description: | + The time when the job ended. + - !ruby/object:Api::Type::Enum + name: 'state' + output: true + description: | + Execution state for the job. + values: + - :STATE_UNSPECIFIED + - :RUNNING + - :CANCELLING + - :CANCELLED + - :SUCCEEDED + - :FAILED + - :ABORTED + - !ruby/object:Api::Type::Integer + name: 'retryCount' + output: true + description: | + The number of times the job has been retried (excluding the initial attempt). + - !ruby/object:Api::Type::Enum + name: 'service' + output: true + description: | + The underlying service running a job. + values: + - :SERVICE_UNSPECIFIED + - :DATAPROC + - !ruby/object:Api::Type::String + name: 'serviceJob' + output: true + description: | + The full resource name for the job run under a particular service. + - !ruby/object:Api::Type::String + name: 'message' + output: true + description: | + Additional information about the current state. + - !ruby/object:Api::Type::NestedObject + name: 'spark' + description: | + A service with manual scaling runs continuously, allowing you to perform complex initialization and rely on the state of its memory over time. + exactly_one_of: + - spark + - notebook + properties: + - !ruby/object:Api::Type::Array + name: 'fileUris' + description: | + Cloud Storage URIs of files to be placed in the working directory of each executor. + item_type: Api::Type::String + - !ruby/object:Api::Type::Array + name: 'archiveUris' + description: | + Cloud Storage URIs of archives to be extracted into the working directory of each executor. Supported file types: .jar, .tar, .tar.gz, .tgz, and .zip. + item_type: Api::Type::String + - !ruby/object:Api::Type::NestedObject + name: 'infrastructureSpec' + description: | + Infrastructure specification for the execution. + properties: + - !ruby/object:Api::Type::NestedObject + name: 'batch' + description: | + Compute resources needed for a Task when using Dataproc Serverless. + properties: + - !ruby/object:Api::Type::Integer + name: 'executorsCount' + default_value: 2 + description: | + Total number of job executors. Executor Count should be between 2 and 100. [Default=2] + - !ruby/object:Api::Type::Integer + name: 'maxExecutorsCount' + default_value: 1000 + description: | + Max configurable executors. If maxExecutorsCount > executorsCount, then auto-scaling is enabled. Max Executor Count should be between 2 and 1000. [Default=1000] + - !ruby/object:Api::Type::NestedObject + name: 'containerImage' + description: | + Container Image Runtime Configuration. + properties: + - !ruby/object:Api::Type::String + name: 'image' + description: | + Container image to use. + - !ruby/object:Api::Type::Array + name: 'javaJars' + description: | + A list of Java JARS to add to the classpath. Valid input includes Cloud Storage URIs to Jar binaries. For example, gs://bucket-name/my/path/to/file.jar + item_type: Api::Type::String + - !ruby/object:Api::Type::Array + name: 'pythonPackages' + description: | + A list of python packages to be installed. Valid formats include Cloud Storage URI to a PIP installable library. For example, gs://bucket-name/my/path/to/lib.tar.gz + item_type: Api::Type::String + - !ruby/object:Api::Type::KeyValuePairs + name: 'properties' + description: | + Override to common configuration of open source components installed on the Dataproc cluster. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. For more information, see Cluster properties. + - !ruby/object:Api::Type::NestedObject + name: 'vpcNetwork' + description: | + Vpc network. + properties: + - !ruby/object:Api::Type::Array + name: 'networkTags' + description: | + List of network tags to apply to the job. + item_type: Api::Type::String + - !ruby/object:Api::Type::String + name: 'network' + description: | + The Cloud VPC network in which the job is run. By default, the Cloud VPC network named Default within the project is used. + exactly_one_of: + - network + - subNetwork + - !ruby/object:Api::Type::String + name: 'subNetwork' + description: | + The Cloud VPC sub-network in which the job is run. + exactly_one_of: + - network + - subNetwork + - !ruby/object:Api::Type::String + name: 'mainJarFileUri' + description: | + The Cloud Storage URI of the jar file that contains the main class. The execution args are passed in as a sequence of named process arguments (--key=value). + exactly_one_of: + - mainJarFileUri + - mainClass + - pythonScriptFile + - sqlScriptFile + - sqlScript + - !ruby/object:Api::Type::String + name: 'mainClass' + description: | + The name of the driver's main class. The jar file that contains the class must be in the default CLASSPATH or specified in jar_file_uris. The execution args are passed in as a sequence of named process arguments (--key=value). + exactly_one_of: + - mainJarFileUri + - mainClass + - pythonScriptFile + - sqlScriptFile + - sqlScript + - !ruby/object:Api::Type::String + name: 'pythonScriptFile' + description: | + The Gcloud Storage URI of the main Python file to use as the driver. Must be a .py file. The execution args are passed in as a sequence of named process arguments (--key=value). + exactly_one_of: + - mainJarFileUri + - mainClass + - pythonScriptFile + - sqlScriptFile + - sqlScript + - !ruby/object:Api::Type::String + name: 'sqlScriptFile' + description: | + A reference to a query file. This can be the Cloud Storage URI of the query file or it can the path to a SqlScript Content. The execution args are used to declare a set of script variables (set key='value';). + exactly_one_of: + - mainJarFileUri + - mainClass + - pythonScriptFile + - sqlScriptFile + - sqlScript + - !ruby/object:Api::Type::String + name: 'sqlScript' + description: | + The query text. The execution args are used to declare a set of script variables (set key='value';). + exactly_one_of: + - mainJarFileUri + - mainClass + - pythonScriptFile + - sqlScriptFile + - sqlScript + - !ruby/object:Api::Type::NestedObject + name: 'notebook' + description: | + A service with manual scaling runs continuously, allowing you to perform complex initialization and rely on the state of its memory over time. + exactly_one_of: + - spark + - notebook + properties: + - !ruby/object:Api::Type::String + name: 'notebook' + required: true + description: | + Path to input notebook. This can be the Cloud Storage URI of the notebook file or the path to a Notebook Content. The execution args are accessible as environment variables (TASK_key=value). + - !ruby/object:Api::Type::NestedObject + name: 'infrastructureSpec' + description: | + Infrastructure specification for the execution. + properties: + - !ruby/object:Api::Type::NestedObject + name: 'batch' + description: | + Compute resources needed for a Task when using Dataproc Serverless. + properties: + - !ruby/object:Api::Type::Integer + name: 'executorsCount' + default_value: 2 + description: | + Total number of job executors. Executor Count should be between 2 and 100. [Default=2] + - !ruby/object:Api::Type::Integer + name: 'maxExecutorsCount' + default_value: 1000 + description: | + Max configurable executors. If maxExecutorsCount > executorsCount, then auto-scaling is enabled. Max Executor Count should be between 2 and 1000. [Default=1000] + - !ruby/object:Api::Type::NestedObject + name: 'containerImage' + description: | + Container Image Runtime Configuration. + properties: + - !ruby/object:Api::Type::String + name: 'image' + description: | + Container image to use. + - !ruby/object:Api::Type::Array + name: 'javaJars' + description: | + A list of Java JARS to add to the classpath. Valid input includes Cloud Storage URIs to Jar binaries. For example, gs://bucket-name/my/path/to/file.jar + item_type: Api::Type::String + - !ruby/object:Api::Type::Array + name: 'pythonPackages' + description: | + A list of python packages to be installed. Valid formats include Cloud Storage URI to a PIP installable library. For example, gs://bucket-name/my/path/to/lib.tar.gz + item_type: Api::Type::String + - !ruby/object:Api::Type::KeyValuePairs + name: 'properties' + description: | + Override to common configuration of open source components installed on the Dataproc cluster. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. For more information, see Cluster properties. + - !ruby/object:Api::Type::NestedObject + name: 'vpcNetwork' + description: | + Vpc network. + properties: + - !ruby/object:Api::Type::Array + name: 'networkTags' + description: | + List of network tags to apply to the job. + item_type: Api::Type::String + - !ruby/object:Api::Type::String + name: 'network' + description: | + The Cloud VPC network in which the job is run. By default, the Cloud VPC network named Default within the project is used. + exactly_one_of: + - network + - subNetwork + - !ruby/object:Api::Type::String + name: 'subNetwork' + description: | + The Cloud VPC sub-network in which the job is run. + exactly_one_of: + - network + - subNetwork + - !ruby/object:Api::Type::Array + name: 'fileUris' + description: | + Cloud Storage URIs of files to be placed in the working directory of each executor. + item_type: Api::Type::String + - !ruby/object:Api::Type::Array + name: 'archiveUris' + description: | + Cloud Storage URIs of archives to be extracted into the working directory of each executor. Supported file types: .jar, .tar, .tar.gz, .tgz, and .zip. + item_type: Api::Type::String +examples: + - !ruby/object:Provider::Terraform::Examples + name: 'dataplex_task_primary' + primary_resource_id: 'example' + primary_resource_name: + 'fmt.Sprintf("tf-test-lake%s", context["random_suffix"]), + fmt.Sprintf("tf-test-task%s", context["random_suffix"])' + test_env_vars: + project_name: :PROJECT_NAME \ No newline at end of file diff --git a/mmv1/products/dataplex/product.yaml b/mmv1/products/dataplex/product.yaml index 1097808e49d6..fd594a264084 100644 --- a/mmv1/products/dataplex/product.yaml +++ b/mmv1/products/dataplex/product.yaml @@ -23,4 +23,4 @@ scopes: apis_required: - !ruby/object:Api::Product::ApiReference name: Cloud Dataplex API - url: https://cloud.google.com/dataplex/docs/reference/rest/ + url: https://console.cloud.google.com/apis/library/dataplex.googleapis.com \ No newline at end of file diff --git a/mmv1/templates/terraform/examples/dataplex_task_primary.tf.erb b/mmv1/templates/terraform/examples/dataplex_task_primary.tf.erb new file mode 100644 index 000000000000..f37eeda85081 --- /dev/null +++ b/mmv1/templates/terraform/examples/dataplex_task_primary.tf.erb @@ -0,0 +1,27 @@ +resource "google_dataplex_lake" "<%= ctx[:primary_resource_id] %>" { + name = "tf-test-lake%{random_suffix}" + location = "us-central1" + project = "<%= ctx[:test_env_vars]['project_name'] %>" +} + + +resource "google_dataplex_task" "<%= ctx[:primary_resource_id] %>" { + + task_id = "tf-test-task%{random_suffix}" + location = "us-central1" + lake = google_dataplex_lake.<%= ctx[:primary_resource_id] %>.name + trigger_spec { + type = "ON_DEMAND" + } + + execution_spec { + service_account = "285091628941-compute@developer.gserviceaccount.com" + } + + spark { + python_script_file = "gs://dataproc-examples/pyspark/hello-world/hello-world.py" + } + + project = "<%= ctx[:test_env_vars]['project_name'] %>" + +} From 958fc2e2e28ce0f25ea15c0b47c5377ff743089a Mon Sep 17 00:00:00 2001 From: sachitjaggi156 Date: Fri, 9 Jun 2023 06:34:26 +0000 Subject: [PATCH 02/19] Changed Async config --- mmv1/products/dataplex/Task.yaml | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/mmv1/products/dataplex/Task.yaml b/mmv1/products/dataplex/Task.yaml index c6d2edac48df..b3ab2a9ee675 100644 --- a/mmv1/products/dataplex/Task.yaml +++ b/mmv1/products/dataplex/Task.yaml @@ -20,9 +20,27 @@ update_url: 'projects/{{project}}/locations/{{location}}/lakes/{{lake}}/tasks/{{ delete_url: 'projects/{{project}}/locations/{{location}}/lakes/{{lake}}/tasks/{{task_id}}' description: | Used to create Dataplex Tasks -async: !ruby/object:Provider::Terraform::PollAsync - check_response_func_existence: PollCheckForExistence - actions: ['create'] +async: !ruby/object:Api::OpAsync + operation: !ruby/object:Api::OpAsync::Operation + path: 'name' + base_url: '{{op_id}}' + wait_ms: 1000 + timeouts: !ruby/object:Api::Timeouts + insert_minutes: 5 + update_minutes: 5 + delete_minutes: 5 + result: !ruby/object:Api::OpAsync::Result + path: 'response' + status: !ruby/object:Api::OpAsync::Status + path: 'done' + complete: true + allowed: + - true + - false + error: !ruby/object:Api::OpAsync::Error + path: 'error' + message: 'message' +autogen_async: true references: !ruby/object:Api::Resource::ReferenceLinks guides: 'Official Documentation': 'https://cloud.google.com/dataplex/docs' From ec7c78e97fd2c7e2983d4027e7c6e0e76bc5eada Mon Sep 17 00:00:00 2001 From: sachitjaggi156 Date: Fri, 9 Jun 2023 06:46:31 +0000 Subject: [PATCH 03/19] Removed Hard coded Service Account --- mmv1/templates/terraform/examples/dataplex_task_primary.tf.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmv1/templates/terraform/examples/dataplex_task_primary.tf.erb b/mmv1/templates/terraform/examples/dataplex_task_primary.tf.erb index f37eeda85081..3c344449aa78 100644 --- a/mmv1/templates/terraform/examples/dataplex_task_primary.tf.erb +++ b/mmv1/templates/terraform/examples/dataplex_task_primary.tf.erb @@ -15,7 +15,7 @@ resource "google_dataplex_task" "<%= ctx[:primary_resource_id] %>" { } execution_spec { - service_account = "285091628941-compute@developer.gserviceaccount.com" + service_account = "<%= ctx[:test_env_vars]['project_number'] %>-compute@developer.gserviceaccount.com" } spark { From 4a547ca64d9f03d86cb19449633a99b3d840bc72 Mon Sep 17 00:00:00 2001 From: sachitjaggi156 Date: Fri, 9 Jun 2023 06:59:28 +0000 Subject: [PATCH 04/19] Used Google Project datablock --- .../terraform/examples/dataplex_task_primary.tf.erb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mmv1/templates/terraform/examples/dataplex_task_primary.tf.erb b/mmv1/templates/terraform/examples/dataplex_task_primary.tf.erb index 3c344449aa78..d46e49698b39 100644 --- a/mmv1/templates/terraform/examples/dataplex_task_primary.tf.erb +++ b/mmv1/templates/terraform/examples/dataplex_task_primary.tf.erb @@ -1,3 +1,7 @@ +data "google_project" "project" { + +} + resource "google_dataplex_lake" "<%= ctx[:primary_resource_id] %>" { name = "tf-test-lake%{random_suffix}" location = "us-central1" @@ -15,7 +19,7 @@ resource "google_dataplex_task" "<%= ctx[:primary_resource_id] %>" { } execution_spec { - service_account = "<%= ctx[:test_env_vars]['project_number'] %>-compute@developer.gserviceaccount.com" + service_account = "${data.google_project.project.number}-compute@developer.gserviceaccount.com" } spark { From e791e8e5bfc155ee3db31388f7a9f4ae4103f4d4 Mon Sep 17 00:00:00 2001 From: sachitjaggi156 Date: Mon, 19 Jun 2023 08:34:08 +0000 Subject: [PATCH 05/19] Resolved comments --- mmv1/products/dataplex/Task.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/mmv1/products/dataplex/Task.yaml b/mmv1/products/dataplex/Task.yaml index b3ab2a9ee675..633ca449b169 100644 --- a/mmv1/products/dataplex/Task.yaml +++ b/mmv1/products/dataplex/Task.yaml @@ -16,10 +16,9 @@ name: 'Task' base_url: 'projects/{{project}}/locations/{{location}}/lakes/{{lake}}/tasks/{{task_id}}' self_link: 'projects/{{project}}/locations/{{location}}/lakes/{{lake}}/tasks/{{task_id}}' create_url: 'projects/{{project}}/locations/{{location}}/lakes/{{lake}}/tasks?task_id={{task_id}}' -update_url: 'projects/{{project}}/locations/{{location}}/lakes/{{lake}}/tasks/{{task_id}}' delete_url: 'projects/{{project}}/locations/{{location}}/lakes/{{lake}}/tasks/{{task_id}}' description: | - Used to create Dataplex Tasks + A Dataplex task represents the work that you want Dataplex to do on a schedule. It encapsulates code, parameters, and the schedule. async: !ruby/object:Api::OpAsync operation: !ruby/object:Api::OpAsync::Operation path: 'name' @@ -44,7 +43,7 @@ autogen_async: true references: !ruby/object:Api::Resource::ReferenceLinks guides: 'Official Documentation': 'https://cloud.google.com/dataplex/docs' - api: 'https://cloud.google.com/dataplex/docs/reference/rest' + api: 'https://cloud.google.com/dataplex/docs/reference/rest/v1/projects.locations.lakes.tasks' import_format: ['projects/{{project}}/locations/{{location}}/lakes/{{lake}}/tasks/{{task_id}}'] iam_policy: !ruby/object:Api::Resource::IamPolicy exclude: false @@ -132,7 +131,6 @@ properties: description: | Trigger type of the user-specified Task values: - - :TYPE_UNSPECIFIED - :ON_DEMAND - :RECURRING - !ruby/object:Api::Type::Time From ce040e90b8c5f96d42eab243c42e67ed3eb2eca2 Mon Sep 17 00:00:00 2001 From: sachitjaggi156 Date: Wed, 21 Jun 2023 21:44:25 +0000 Subject: [PATCH 06/19] Update Test Case --- mmv1/products/dataplex/Task.yaml | 1 + .../tests/resource_dataplex_task_test.go | 177 ++++++++++++++++++ 2 files changed, 178 insertions(+) create mode 100644 mmv1/third_party/terraform/tests/resource_dataplex_task_test.go diff --git a/mmv1/products/dataplex/Task.yaml b/mmv1/products/dataplex/Task.yaml index 633ca449b169..8eed76b53e76 100644 --- a/mmv1/products/dataplex/Task.yaml +++ b/mmv1/products/dataplex/Task.yaml @@ -16,6 +16,7 @@ name: 'Task' base_url: 'projects/{{project}}/locations/{{location}}/lakes/{{lake}}/tasks/{{task_id}}' self_link: 'projects/{{project}}/locations/{{location}}/lakes/{{lake}}/tasks/{{task_id}}' create_url: 'projects/{{project}}/locations/{{location}}/lakes/{{lake}}/tasks?task_id={{task_id}}' +update_url: 'projects/{{project}}/locations/{{location}}/lakes/{{lake}}/tasks/{{task_id}}' delete_url: 'projects/{{project}}/locations/{{location}}/lakes/{{lake}}/tasks/{{task_id}}' description: | A Dataplex task represents the work that you want Dataplex to do on a schedule. It encapsulates code, parameters, and the schedule. diff --git a/mmv1/third_party/terraform/tests/resource_dataplex_task_test.go b/mmv1/third_party/terraform/tests/resource_dataplex_task_test.go new file mode 100644 index 000000000000..3b21c1021ae3 --- /dev/null +++ b/mmv1/third_party/terraform/tests/resource_dataplex_task_test.go @@ -0,0 +1,177 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +// ---------------------------------------------------------------------------- +// +// *** AUTO GENERATED CODE *** Type: MMv1 *** +// +// ---------------------------------------------------------------------------- +// +// This file is automatically generated by Magic Modules and manual +// changes will be clobbered when the file is regenerated. +// +// Please read more about how to change this file in +// .github/CONTRIBUTING.md. +// +// ---------------------------------------------------------------------------- + +package google + +import ( + "fmt" + "strings" + "testing" + + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" + "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" + + "github.com/hashicorp/terraform-provider-google/google/acctest" + "github.com/hashicorp/terraform-provider-google/google/tpgresource" + transport_tpg "github.com/hashicorp/terraform-provider-google/google/transport" +) + +func TestAccDataplexTaskDataplexTaskPrimary_update(t *testing.T) { + t.Parallel() + + context := map[string]interface{}{ + "project_name": acctest.GetTestProjectFromEnv(), + "random_suffix": RandString(t, 10), + } + + VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: ProtoV5ProviderFactories(t), + CheckDestroy: testAccCheckDataplexTaskDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccDataplexTask_dataplexTaskPrimary(context), + }, + { + ResourceName: "google_dataplex_task.example", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"location", "lake", "task_id"}, + }, + { + Config: testAccDataplexTask_dataplexTaskPrimaryUpdate(context), + }, + { + ResourceName: "google_dataplex_task.example", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"location", "lake", "task_id"}, + }, + }, + }) +} + +func testAccDataplexTask_dataplexTaskPrimary(context map[string]interface{}) string { + return tpgresource.Nprintf(` +data "google_project" "project" { + +} + +resource "google_dataplex_lake" "example" { + name = "tf-test-lake%{random_suffix}" + location = "us-central1" + project = "%{project_name}" +} + + +resource "google_dataplex_task" "example" { + + task_id = "tf-test-task%{random_suffix}" + location = "us-central1" + lake = google_dataplex_lake.example.name + trigger_spec { + type = "ON_DEMAND" + } + + execution_spec { + service_account = "${data.google_project.project.number}-compute@developer.gserviceaccount.com" + } + + spark { + python_script_file = "gs://dataproc-examples/pyspark/hello-world/hello-world.py" + } + + project = "%{project_name}" + +} +`, context) +} + +func testAccDataplexTask_dataplexTaskPrimaryUpdate(context map[string]interface{}) string { + return tpgresource.Nprintf(` +data "google_project" "project" { + +} + +resource "google_dataplex_lake" "example" { + name = "tf-test-lake%{random_suffix}" + location = "us-central1" + project = "%{project_name}" +} + + +resource "google_dataplex_task" "example" { + + task_id = "tf-test-task%{random_suffix}" + location = "us-central1" + lake = google_dataplex_lake.example.name + trigger_spec { + type = "ON_DEMAND" + } + + execution_spec { + service_account = "${data.google_project.project.number}-compute@developer.gserviceaccount.com" + } + + spark { + python_script_file = "gs://dataplex-clouddq-api-integration-test/clouddq_pyspark_driver.py" + } + + project = "%{project_name}" + +} +`, context) +} + +func testAccCheckDataplexTaskDestroyProducer(t *testing.T) func(s *terraform.State) error { + return func(s *terraform.State) error { + for name, rs := range s.RootModule().Resources { + if rs.Type != "google_dataplex_task" { + continue + } + if strings.HasPrefix(name, "data.") { + continue + } + + config := GoogleProviderConfig(t) + + url, err := tpgresource.ReplaceVarsForTest(config, rs, "{{DataplexBasePath}}projects/{{project}}/locations/{{location}}/lakes/{{lake}}/tasks/{{task_id}}") + if err != nil { + return err + } + + billingProject := "" + + if config.BillingProject != "" { + billingProject = config.BillingProject + } + + _, err = transport_tpg.SendRequest(transport_tpg.SendRequestOptions{ + Config: config, + Method: "GET", + Project: billingProject, + RawURL: url, + UserAgent: config.UserAgent, + }) + if err == nil { + return fmt.Errorf("DataplexTask still exists at %s", url) + } + } + + return nil + } +} From c4999d14e771b8e3a33c6a96c83674f2addfe518 Mon Sep 17 00:00:00 2001 From: sachitjaggi156 Date: Mon, 26 Jun 2023 09:52:18 +0000 Subject: [PATCH 07/19] Update Test case and I am Policy testcase fix --- mmv1/products/dataplex/Task.yaml | 6 ++- ...mary.tf.erb => dataplex_task_basic.tf.erb} | 0 .../tests/resource_dataplex_task_test.go | 44 +------------------ 3 files changed, 5 insertions(+), 45 deletions(-) rename mmv1/templates/terraform/examples/{dataplex_task_primary.tf.erb => dataplex_task_basic.tf.erb} (100%) diff --git a/mmv1/products/dataplex/Task.yaml b/mmv1/products/dataplex/Task.yaml index 8eed76b53e76..204aa06e4336 100644 --- a/mmv1/products/dataplex/Task.yaml +++ b/mmv1/products/dataplex/Task.yaml @@ -18,6 +18,8 @@ self_link: 'projects/{{project}}/locations/{{location}}/lakes/{{lake}}/tasks/{{t create_url: 'projects/{{project}}/locations/{{location}}/lakes/{{lake}}/tasks?task_id={{task_id}}' update_url: 'projects/{{project}}/locations/{{location}}/lakes/{{lake}}/tasks/{{task_id}}' delete_url: 'projects/{{project}}/locations/{{location}}/lakes/{{lake}}/tasks/{{task_id}}' +update_verb: :PATCH +update_mask: true description: | A Dataplex task represents the work that you want Dataplex to do on a schedule. It encapsulates code, parameters, and the schedule. async: !ruby/object:Api::OpAsync @@ -49,7 +51,7 @@ import_format: ['projects/{{project}}/locations/{{location}}/lakes/{{lake}}/task iam_policy: !ruby/object:Api::Resource::IamPolicy exclude: false method_name_separator: ':' - parent_resource_attribute: 'task' + parent_resource_attribute: 'task_id' fetch_iam_policy_verb: :POST import_format: [ @@ -477,7 +479,7 @@ properties: item_type: Api::Type::String examples: - !ruby/object:Provider::Terraform::Examples - name: 'dataplex_task_primary' + name: 'dataplex_task_basic' primary_resource_id: 'example' primary_resource_name: 'fmt.Sprintf("tf-test-lake%s", context["random_suffix"]), diff --git a/mmv1/templates/terraform/examples/dataplex_task_primary.tf.erb b/mmv1/templates/terraform/examples/dataplex_task_basic.tf.erb similarity index 100% rename from mmv1/templates/terraform/examples/dataplex_task_primary.tf.erb rename to mmv1/templates/terraform/examples/dataplex_task_basic.tf.erb diff --git a/mmv1/third_party/terraform/tests/resource_dataplex_task_test.go b/mmv1/third_party/terraform/tests/resource_dataplex_task_test.go index 3b21c1021ae3..67e98bed0adb 100644 --- a/mmv1/third_party/terraform/tests/resource_dataplex_task_test.go +++ b/mmv1/third_party/terraform/tests/resource_dataplex_task_test.go @@ -18,19 +18,15 @@ package google import ( - "fmt" - "strings" "testing" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" - "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" "github.com/hashicorp/terraform-provider-google/google/acctest" "github.com/hashicorp/terraform-provider-google/google/tpgresource" - transport_tpg "github.com/hashicorp/terraform-provider-google/google/transport" ) -func TestAccDataplexTaskDataplexTaskPrimary_update(t *testing.T) { +func TestAccDataplexTaskDataplexTask_update(t *testing.T) { t.Parallel() context := map[string]interface{}{ @@ -137,41 +133,3 @@ resource "google_dataplex_task" "example" { `, context) } -func testAccCheckDataplexTaskDestroyProducer(t *testing.T) func(s *terraform.State) error { - return func(s *terraform.State) error { - for name, rs := range s.RootModule().Resources { - if rs.Type != "google_dataplex_task" { - continue - } - if strings.HasPrefix(name, "data.") { - continue - } - - config := GoogleProviderConfig(t) - - url, err := tpgresource.ReplaceVarsForTest(config, rs, "{{DataplexBasePath}}projects/{{project}}/locations/{{location}}/lakes/{{lake}}/tasks/{{task_id}}") - if err != nil { - return err - } - - billingProject := "" - - if config.BillingProject != "" { - billingProject = config.BillingProject - } - - _, err = transport_tpg.SendRequest(transport_tpg.SendRequestOptions{ - Config: config, - Method: "GET", - Project: billingProject, - RawURL: url, - UserAgent: config.UserAgent, - }) - if err == nil { - return fmt.Errorf("DataplexTask still exists at %s", url) - } - } - - return nil - } -} From 9777eabf1d4abd6291d271f53183c73c88dca638 Mon Sep 17 00:00:00 2001 From: sachitjaggi156 Date: Mon, 26 Jun 2023 18:14:21 +0000 Subject: [PATCH 08/19] Added Coverage Tests --- mmv1/products/dataplex/Task.yaml | 16 ++++++++ .../examples/dataplex_task_notebook.tf.erb | 32 ++++++++++++++++ .../examples/dataplex_task_spark.tf.erb | 37 +++++++++++++++++++ 3 files changed, 85 insertions(+) create mode 100644 mmv1/templates/terraform/examples/dataplex_task_notebook.tf.erb create mode 100644 mmv1/templates/terraform/examples/dataplex_task_spark.tf.erb diff --git a/mmv1/products/dataplex/Task.yaml b/mmv1/products/dataplex/Task.yaml index 204aa06e4336..bc30779bae6f 100644 --- a/mmv1/products/dataplex/Task.yaml +++ b/mmv1/products/dataplex/Task.yaml @@ -481,6 +481,22 @@ examples: - !ruby/object:Provider::Terraform::Examples name: 'dataplex_task_basic' primary_resource_id: 'example' + primary_resource_name: + 'fmt.Sprintf("tf-test-lake%s", context["random_suffix"]), + fmt.Sprintf("tf-test-task%s", context["random_suffix"])' + test_env_vars: + project_name: :PROJECT_NAME + - !ruby/object:Provider::Terraform::Examples + name: 'dataplex_task_spark' + primary_resource_id: 'example_spark' + primary_resource_name: + 'fmt.Sprintf("tf-test-lake%s", context["random_suffix"]), + fmt.Sprintf("tf-test-task%s", context["random_suffix"])' + test_env_vars: + project_name: :PROJECT_NAME + - !ruby/object:Provider::Terraform::Examples + name: 'dataplex_task_notebook' + primary_resource_id: 'example_notebook' primary_resource_name: 'fmt.Sprintf("tf-test-lake%s", context["random_suffix"]), fmt.Sprintf("tf-test-task%s", context["random_suffix"])' diff --git a/mmv1/templates/terraform/examples/dataplex_task_notebook.tf.erb b/mmv1/templates/terraform/examples/dataplex_task_notebook.tf.erb new file mode 100644 index 000000000000..c8a1ff573721 --- /dev/null +++ b/mmv1/templates/terraform/examples/dataplex_task_notebook.tf.erb @@ -0,0 +1,32 @@ +data "google_project" "project" { + +} + +resource "google_dataplex_lake" "<%= ctx[:primary_resource_id] %>" { + name = "tf-test-lake%{random_suffix}" + location = "us-central1" + project = "<%= ctx[:test_env_vars]['project_name'] %>" +} + + +resource "google_dataplex_task" "<%= ctx[:primary_resource_id] %>" { + + task_id = "tf-test-task%{random_suffix}" + location = "us-central1" + lake = google_dataplex_lake.<%= ctx[:primary_resource_id] %>.name + trigger_spec { + type = "RECURRING" + schedule = "1 * * * *" + } + + execution_spec { + service_account = "${data.google_project.project.number}-compute@developer.gserviceaccount.com" + } + + spark { + python_script_file = "gs://dataproc-examples/pyspark/hello-world/hello-world.py" + } + + project = "<%= ctx[:test_env_vars]['project_name'] %>" + +} diff --git a/mmv1/templates/terraform/examples/dataplex_task_spark.tf.erb b/mmv1/templates/terraform/examples/dataplex_task_spark.tf.erb new file mode 100644 index 000000000000..6b0b35215170 --- /dev/null +++ b/mmv1/templates/terraform/examples/dataplex_task_spark.tf.erb @@ -0,0 +1,37 @@ +data "google_project" "project" { + +} + +resource "google_dataplex_lake" "<%= ctx[:primary_resource_id] %>" { + name = "tf-test-lake%{random_suffix}" + location = "us-central1" + project = "<%= ctx[:test_env_vars]['project_name'] %>" +} + + +resource "google_dataplex_task" "<%= ctx[:primary_resource_id] %>" { + + task_id = "tf-test-task%{random_suffix}" + location = "us-central1" + lake = google_dataplex_lake.<%= ctx[:primary_resource_id] %>.name + trigger_spec { + type = "ON_DEMAND" + } + + description = "task-spark-terraform" + + execution_spec { + service_account = "${data.google_project.project.number}-compute@developer.gserviceaccount.com" + args = { + TASK_ARGS = "--output_location,gs://spark-job-jars-anrajitha/task-result, --output_format, json" + } + + } + + spark { + sql_script = "show databases" + } + + project = "<%= ctx[:test_env_vars]['project_name'] %>" + +} From e4db54020ec2641b5e788095c4f26d31e10dae72 Mon Sep 17 00:00:00 2001 From: sachitjaggi156 Date: Sun, 2 Jul 2023 21:55:27 +0000 Subject: [PATCH 09/19] Added test-cases for remaining fields --- .../examples/dataplex_task_basic.tf.erb | 9 +++++- .../examples/dataplex_task_notebook.tf.erb | 29 ++++++++++++++++--- .../examples/dataplex_task_spark.tf.erb | 18 ++++++++++++ 3 files changed, 51 insertions(+), 5 deletions(-) diff --git a/mmv1/templates/terraform/examples/dataplex_task_basic.tf.erb b/mmv1/templates/terraform/examples/dataplex_task_basic.tf.erb index d46e49698b39..8cb3897b67a4 100644 --- a/mmv1/templates/terraform/examples/dataplex_task_basic.tf.erb +++ b/mmv1/templates/terraform/examples/dataplex_task_basic.tf.erb @@ -15,11 +15,18 @@ resource "google_dataplex_task" "<%= ctx[:primary_resource_id] %>" { location = "us-central1" lake = google_dataplex_lake.<%= ctx[:primary_resource_id] %>.name trigger_spec { - type = "ON_DEMAND" + type = "RECURRING" + disabled = false + max_retries = 3 + start_time = "2023-10-02T15:01:23Z" + schedule = "1 * * * *" } execution_spec { service_account = "${data.google_project.project.number}-compute@developer.gserviceaccount.com" + project = "<%= ctx[:test_env_vars]['project_name'] %>" + max_job_execution_lifetime = "100s" + kms_key = "234jn2kjn42k3n423" } spark { diff --git a/mmv1/templates/terraform/examples/dataplex_task_notebook.tf.erb b/mmv1/templates/terraform/examples/dataplex_task_notebook.tf.erb index c8a1ff573721..38463aab5a59 100644 --- a/mmv1/templates/terraform/examples/dataplex_task_notebook.tf.erb +++ b/mmv1/templates/terraform/examples/dataplex_task_notebook.tf.erb @@ -21,12 +21,33 @@ resource "google_dataplex_task" "<%= ctx[:primary_resource_id] %>" { execution_spec { service_account = "${data.google_project.project.number}-compute@developer.gserviceaccount.com" + args = { + TASK_ARGS = "--output_location,gs://spark-job-jars-anrajitha/task-result, --output_format, json" + } } - - spark { - python_script_file = "gs://dataproc-examples/pyspark/hello-world/hello-world.py" + notebook { + notebook = "gs://ssmadhavik-test4/Demo/CarPricePrediction_OLS_ML.ipynb" + infrastructure_spec { + batch { + executors_count = 2 + max_executors_count = 100 + } + container_image { + image = "test-image" + java_jars = ["test-java-jars.jar"] + python_packages = ["gs://bucket-name/my/path/to/lib.tar.gz"] + properties = { "name": "wrench", "mass": "1.3kg", "count": "3" } + } + vpc_network { + network_tags = ["test-network-tag"] + network = "test-network" + } + } + file_uris = ["gs://ssmadhavik-test4/Demo/CarPrice.csv"] + archive_uris = ["gs://ssmadhavik-test4/Demo/CarPrice.csv"] + } - project = "<%= ctx[:test_env_vars]['project_name'] %>" + } diff --git a/mmv1/templates/terraform/examples/dataplex_task_spark.tf.erb b/mmv1/templates/terraform/examples/dataplex_task_spark.tf.erb index 6b0b35215170..78e1fe668323 100644 --- a/mmv1/templates/terraform/examples/dataplex_task_spark.tf.erb +++ b/mmv1/templates/terraform/examples/dataplex_task_spark.tf.erb @@ -29,6 +29,24 @@ resource "google_dataplex_task" "<%= ctx[:primary_resource_id] %>" { } spark { + infrastructure_spec { + batch { + executors_count = 2 + max_executors_count = 100 + } + container_image { + image = "test-image" + java_jars = ["test-java-jars.jar"] + python_packages = ["gs://bucket-name/my/path/to/lib.tar.gz"] + properties = { "name": "wrench", "mass": "1.3kg", "count": "3" } + } + vpc_network { + network_tags = ["test-network-tag"] + sub_network = "test-network" + } + } + file_uris = ["gs://ssmadhavik-test4/Demo/CarPrice.csv"] + archive_uris = ["gs://ssmadhavik-test4/Demo/CarPrice.csv"] sql_script = "show databases" } From bfbafc1f3a1b1b7c64fd1f6e3144d7269a4a810e Mon Sep 17 00:00:00 2001 From: sachitjaggi156 Date: Sun, 2 Jul 2023 22:23:12 +0000 Subject: [PATCH 10/19] Added a few fields to test-cases --- .../terraform/examples/dataplex_task_basic.tf.erb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mmv1/templates/terraform/examples/dataplex_task_basic.tf.erb b/mmv1/templates/terraform/examples/dataplex_task_basic.tf.erb index 8cb3897b67a4..e5cafb894773 100644 --- a/mmv1/templates/terraform/examples/dataplex_task_basic.tf.erb +++ b/mmv1/templates/terraform/examples/dataplex_task_basic.tf.erb @@ -14,6 +14,13 @@ resource "google_dataplex_task" "<%= ctx[:primary_resource_id] %>" { task_id = "tf-test-task%{random_suffix}" location = "us-central1" lake = google_dataplex_lake.<%= ctx[:primary_resource_id] %>.name + + description = "Test Task Basic" + + display_name = "task-basic" + + labels = { "count": "3" } + trigger_spec { type = "RECURRING" disabled = false From 647b7bac02c1462ef5fbbe779f5d04716f36dc7c Mon Sep 17 00:00:00 2001 From: sachitjaggi156 Date: Thu, 6 Jul 2023 07:42:19 +0000 Subject: [PATCH 11/19] Required changes in the basic-example and Update test-case. --- mmv1/templates/terraform/examples/dataplex_task_basic.tf.erb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mmv1/templates/terraform/examples/dataplex_task_basic.tf.erb b/mmv1/templates/terraform/examples/dataplex_task_basic.tf.erb index e5cafb894773..8ba90a5111fd 100644 --- a/mmv1/templates/terraform/examples/dataplex_task_basic.tf.erb +++ b/mmv1/templates/terraform/examples/dataplex_task_basic.tf.erb @@ -38,6 +38,11 @@ resource "google_dataplex_task" "<%= ctx[:primary_resource_id] %>" { spark { python_script_file = "gs://dataproc-examples/pyspark/hello-world/hello-world.py" + + vpc_network { + network_tags = ["test-network-tag"] + sub_network = "test-network" + } } project = "<%= ctx[:test_env_vars]['project_name'] %>" From 659fdafe06536dc5bb4a3d69dd5c99f0ac0c5602 Mon Sep 17 00:00:00 2001 From: sachitjaggi156 Date: Fri, 7 Jul 2023 08:11:55 +0000 Subject: [PATCH 12/19] Checking tests --- .../templates/terraform/examples/dataplex_task_basic.tf.erb | 6 +----- .../terraform/examples/dataplex_task_notebook.tf.erb | 2 +- .../templates/terraform/examples/dataplex_task_spark.tf.erb | 2 +- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/mmv1/templates/terraform/examples/dataplex_task_basic.tf.erb b/mmv1/templates/terraform/examples/dataplex_task_basic.tf.erb index 8ba90a5111fd..c105741576c1 100644 --- a/mmv1/templates/terraform/examples/dataplex_task_basic.tf.erb +++ b/mmv1/templates/terraform/examples/dataplex_task_basic.tf.erb @@ -39,12 +39,8 @@ resource "google_dataplex_task" "<%= ctx[:primary_resource_id] %>" { spark { python_script_file = "gs://dataproc-examples/pyspark/hello-world/hello-world.py" - vpc_network { - network_tags = ["test-network-tag"] - sub_network = "test-network" - } } project = "<%= ctx[:test_env_vars]['project_name'] %>" -} +} \ No newline at end of file diff --git a/mmv1/templates/terraform/examples/dataplex_task_notebook.tf.erb b/mmv1/templates/terraform/examples/dataplex_task_notebook.tf.erb index 38463aab5a59..95f56026695d 100644 --- a/mmv1/templates/terraform/examples/dataplex_task_notebook.tf.erb +++ b/mmv1/templates/terraform/examples/dataplex_task_notebook.tf.erb @@ -50,4 +50,4 @@ resource "google_dataplex_task" "<%= ctx[:primary_resource_id] %>" { project = "<%= ctx[:test_env_vars]['project_name'] %>" -} +} \ No newline at end of file diff --git a/mmv1/templates/terraform/examples/dataplex_task_spark.tf.erb b/mmv1/templates/terraform/examples/dataplex_task_spark.tf.erb index 78e1fe668323..07b6f54fec70 100644 --- a/mmv1/templates/terraform/examples/dataplex_task_spark.tf.erb +++ b/mmv1/templates/terraform/examples/dataplex_task_spark.tf.erb @@ -52,4 +52,4 @@ resource "google_dataplex_task" "<%= ctx[:primary_resource_id] %>" { project = "<%= ctx[:test_env_vars]['project_name'] %>" -} +} \ No newline at end of file From 9dc2bea9c950563ac3fdda0205c06b265eda7e64 Mon Sep 17 00:00:00 2001 From: sachitjaggi156 Date: Fri, 7 Jul 2023 08:23:53 +0000 Subject: [PATCH 13/19] Fix Lint issues --- mmv1/products/dataplex/Task.yaml | 6 +++--- mmv1/products/dataplex/product.yaml | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/mmv1/products/dataplex/Task.yaml b/mmv1/products/dataplex/Task.yaml index bc30779bae6f..d56a2ddda3de 100644 --- a/mmv1/products/dataplex/Task.yaml +++ b/mmv1/products/dataplex/Task.yaml @@ -10,7 +10,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - --- !ruby/object:Api::Resource name: 'Task' base_url: 'projects/{{project}}/locations/{{location}}/lakes/{{lake}}/tasks/{{task_id}}' @@ -466,7 +465,7 @@ properties: The Cloud VPC sub-network in which the job is run. exactly_one_of: - network - - subNetwork + - subNetwork - !ruby/object:Api::Type::Array name: 'fileUris' description: | @@ -501,4 +500,5 @@ examples: 'fmt.Sprintf("tf-test-lake%s", context["random_suffix"]), fmt.Sprintf("tf-test-task%s", context["random_suffix"])' test_env_vars: - project_name: :PROJECT_NAME \ No newline at end of file + project_name: :PROJECT_NAME + \ No newline at end of file diff --git a/mmv1/products/dataplex/product.yaml b/mmv1/products/dataplex/product.yaml index fd594a264084..caee1f05283a 100644 --- a/mmv1/products/dataplex/product.yaml +++ b/mmv1/products/dataplex/product.yaml @@ -23,4 +23,5 @@ scopes: apis_required: - !ruby/object:Api::Product::ApiReference name: Cloud Dataplex API - url: https://console.cloud.google.com/apis/library/dataplex.googleapis.com \ No newline at end of file + url: https://console.cloud.google.com/apis/library/dataplex.googleapis.com + \ No newline at end of file From a8c7ffa4db0cbc4338aca30fb589c5081b9f2a98 Mon Sep 17 00:00:00 2001 From: sachitjaggi156 Date: Sun, 9 Jul 2023 19:42:12 +0000 Subject: [PATCH 14/19] Added Network and fixed lint issues --- .../terraform/examples/dataplex_task_notebook.tf.erb | 9 ++++++++- .../terraform/examples/dataplex_task_spark.tf.erb | 8 +++++++- .../terraform/tests/resource_dataplex_task_test.go | 1 - 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/mmv1/templates/terraform/examples/dataplex_task_notebook.tf.erb b/mmv1/templates/terraform/examples/dataplex_task_notebook.tf.erb index 95f56026695d..6a6847833cde 100644 --- a/mmv1/templates/terraform/examples/dataplex_task_notebook.tf.erb +++ b/mmv1/templates/terraform/examples/dataplex_task_notebook.tf.erb @@ -1,3 +1,10 @@ +# VPC network +resource "google_compute_network" "default" { + name = "tf-test-workstation-cluster%{random_suffix}" + auto_create_subnetworks = true +} + + data "google_project" "project" { } @@ -40,7 +47,7 @@ resource "google_dataplex_task" "<%= ctx[:primary_resource_id] %>" { } vpc_network { network_tags = ["test-network-tag"] - network = "test-network" + network = google_compute_network.default.id } } file_uris = ["gs://ssmadhavik-test4/Demo/CarPrice.csv"] diff --git a/mmv1/templates/terraform/examples/dataplex_task_spark.tf.erb b/mmv1/templates/terraform/examples/dataplex_task_spark.tf.erb index 07b6f54fec70..5ea45acca29e 100644 --- a/mmv1/templates/terraform/examples/dataplex_task_spark.tf.erb +++ b/mmv1/templates/terraform/examples/dataplex_task_spark.tf.erb @@ -1,3 +1,9 @@ +# VPC network +resource "google_compute_network" "default" { + name = "tf-test-workstation-cluster%{random_suffix}" + auto_create_subnetworks = true +} + data "google_project" "project" { } @@ -42,7 +48,7 @@ resource "google_dataplex_task" "<%= ctx[:primary_resource_id] %>" { } vpc_network { network_tags = ["test-network-tag"] - sub_network = "test-network" + sub_network = google_compute_network.default.id } } file_uris = ["gs://ssmadhavik-test4/Demo/CarPrice.csv"] diff --git a/mmv1/third_party/terraform/tests/resource_dataplex_task_test.go b/mmv1/third_party/terraform/tests/resource_dataplex_task_test.go index 67e98bed0adb..a7e7c61a31f7 100644 --- a/mmv1/third_party/terraform/tests/resource_dataplex_task_test.go +++ b/mmv1/third_party/terraform/tests/resource_dataplex_task_test.go @@ -132,4 +132,3 @@ resource "google_dataplex_task" "example" { } `, context) } - From 82b3caee517803677d1c93627e19917a77e98213 Mon Sep 17 00:00:00 2001 From: sachitjaggi156 Date: Sun, 9 Jul 2023 19:46:19 +0000 Subject: [PATCH 15/19] Fixed lint issues and required changes in update test-case --- .../terraform/tests/resource_dataplex_task_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mmv1/third_party/terraform/tests/resource_dataplex_task_test.go b/mmv1/third_party/terraform/tests/resource_dataplex_task_test.go index a7e7c61a31f7..cc4ff4cee2f2 100644 --- a/mmv1/third_party/terraform/tests/resource_dataplex_task_test.go +++ b/mmv1/third_party/terraform/tests/resource_dataplex_task_test.go @@ -23,7 +23,6 @@ import ( "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" "github.com/hashicorp/terraform-provider-google/google/acctest" - "github.com/hashicorp/terraform-provider-google/google/tpgresource" ) func TestAccDataplexTaskDataplexTask_update(t *testing.T) { @@ -62,7 +61,7 @@ func TestAccDataplexTaskDataplexTask_update(t *testing.T) { } func testAccDataplexTask_dataplexTaskPrimary(context map[string]interface{}) string { - return tpgresource.Nprintf(` + return acctest.Nprintf(` data "google_project" "project" { } @@ -98,7 +97,7 @@ resource "google_dataplex_task" "example" { } func testAccDataplexTask_dataplexTaskPrimaryUpdate(context map[string]interface{}) string { - return tpgresource.Nprintf(` + return acctest.Nprintf(` data "google_project" "project" { } From 86f0fd23dff96f9bd9db39004e49b0ca8178d641 Mon Sep 17 00:00:00 2001 From: sachitjaggi156 Date: Sun, 9 Jul 2023 19:54:14 +0000 Subject: [PATCH 16/19] Fixed lint issues --- mmv1/products/dataplex/Task.yaml | 19 +++++++++---------- mmv1/products/dataplex/product.yaml | 1 - 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/mmv1/products/dataplex/Task.yaml b/mmv1/products/dataplex/Task.yaml index d56a2ddda3de..1a5649d0daea 100644 --- a/mmv1/products/dataplex/Task.yaml +++ b/mmv1/products/dataplex/Task.yaml @@ -337,15 +337,15 @@ properties: - network - subNetwork - !ruby/object:Api::Type::String - name: 'mainJarFileUri' - description: | - The Cloud Storage URI of the jar file that contains the main class. The execution args are passed in as a sequence of named process arguments (--key=value). - exactly_one_of: - - mainJarFileUri - - mainClass - - pythonScriptFile - - sqlScriptFile - - sqlScript + name: 'mainJarFileUri' + description: | + The Cloud Storage URI of the jar file that contains the main class. The execution args are passed in as a sequence of named process arguments (--key=value). + exactly_one_of: + - mainJarFileUri + - mainClass + - pythonScriptFile + - sqlScriptFile + - sqlScript - !ruby/object:Api::Type::String name: 'mainClass' description: | @@ -501,4 +501,3 @@ examples: fmt.Sprintf("tf-test-task%s", context["random_suffix"])' test_env_vars: project_name: :PROJECT_NAME - \ No newline at end of file diff --git a/mmv1/products/dataplex/product.yaml b/mmv1/products/dataplex/product.yaml index caee1f05283a..7e6dea430ca9 100644 --- a/mmv1/products/dataplex/product.yaml +++ b/mmv1/products/dataplex/product.yaml @@ -24,4 +24,3 @@ apis_required: - !ruby/object:Api::Product::ApiReference name: Cloud Dataplex API url: https://console.cloud.google.com/apis/library/dataplex.googleapis.com - \ No newline at end of file From b31c0ecaf7eb67028a3fe3341129f782cf0f9dc9 Mon Sep 17 00:00:00 2001 From: sachitjaggi156 Date: Wed, 12 Jul 2023 10:42:19 +0000 Subject: [PATCH 17/19] Resolved Update Verb and made examples generic --- mmv1/products/dataplex/Task.yaml | 2 +- .../terraform/examples/dataplex_task_notebook.tf.erb | 6 +++--- .../templates/terraform/examples/dataplex_task_spark.tf.erb | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/mmv1/products/dataplex/Task.yaml b/mmv1/products/dataplex/Task.yaml index 1a5649d0daea..b431fc6b8d72 100644 --- a/mmv1/products/dataplex/Task.yaml +++ b/mmv1/products/dataplex/Task.yaml @@ -17,7 +17,7 @@ self_link: 'projects/{{project}}/locations/{{location}}/lakes/{{lake}}/tasks/{{t create_url: 'projects/{{project}}/locations/{{location}}/lakes/{{lake}}/tasks?task_id={{task_id}}' update_url: 'projects/{{project}}/locations/{{location}}/lakes/{{lake}}/tasks/{{task_id}}' delete_url: 'projects/{{project}}/locations/{{location}}/lakes/{{lake}}/tasks/{{task_id}}' -update_verb: :PATCH +update_verb: :GET update_mask: true description: | A Dataplex task represents the work that you want Dataplex to do on a schedule. It encapsulates code, parameters, and the schedule. diff --git a/mmv1/templates/terraform/examples/dataplex_task_notebook.tf.erb b/mmv1/templates/terraform/examples/dataplex_task_notebook.tf.erb index 6a6847833cde..d90461079c62 100644 --- a/mmv1/templates/terraform/examples/dataplex_task_notebook.tf.erb +++ b/mmv1/templates/terraform/examples/dataplex_task_notebook.tf.erb @@ -33,7 +33,7 @@ resource "google_dataplex_task" "<%= ctx[:primary_resource_id] %>" { } } notebook { - notebook = "gs://ssmadhavik-test4/Demo/CarPricePrediction_OLS_ML.ipynb" + notebook = "gs://terraform-test4/test-notebook.ipynb" infrastructure_spec { batch { executors_count = 2 @@ -50,8 +50,8 @@ resource "google_dataplex_task" "<%= ctx[:primary_resource_id] %>" { network = google_compute_network.default.id } } - file_uris = ["gs://ssmadhavik-test4/Demo/CarPrice.csv"] - archive_uris = ["gs://ssmadhavik-test4/Demo/CarPrice.csv"] + file_uris = ["gs://terraform-test/test.csv"] + archive_uris = ["gs://terraform-test/test.csv"] } project = "<%= ctx[:test_env_vars]['project_name'] %>" diff --git a/mmv1/templates/terraform/examples/dataplex_task_spark.tf.erb b/mmv1/templates/terraform/examples/dataplex_task_spark.tf.erb index 5ea45acca29e..25363788007d 100644 --- a/mmv1/templates/terraform/examples/dataplex_task_spark.tf.erb +++ b/mmv1/templates/terraform/examples/dataplex_task_spark.tf.erb @@ -29,7 +29,7 @@ resource "google_dataplex_task" "<%= ctx[:primary_resource_id] %>" { execution_spec { service_account = "${data.google_project.project.number}-compute@developer.gserviceaccount.com" args = { - TASK_ARGS = "--output_location,gs://spark-job-jars-anrajitha/task-result, --output_format, json" + TASK_ARGS = "--output_location,gs://spark-job/task-result, --output_format, json" } } @@ -51,8 +51,8 @@ resource "google_dataplex_task" "<%= ctx[:primary_resource_id] %>" { sub_network = google_compute_network.default.id } } - file_uris = ["gs://ssmadhavik-test4/Demo/CarPrice.csv"] - archive_uris = ["gs://ssmadhavik-test4/Demo/CarPrice.csv"] + file_uris = ["gs://terrafrom-test/test.csv"] + archive_uris = ["gs://terraform-test/test.csv"] sql_script = "show databases" } From 4ff09abbd528c8a61e1c5028948c0b423fa6151a Mon Sep 17 00:00:00 2001 From: sachitjaggi156 Date: Thu, 13 Jul 2023 07:51:02 +0000 Subject: [PATCH 18/19] Updated the correct verb --- mmv1/products/dataplex/Task.yaml | 4 ++-- .../terraform/examples/dataplex_task_notebook.tf.erb | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmv1/products/dataplex/Task.yaml b/mmv1/products/dataplex/Task.yaml index b431fc6b8d72..2abf68fb5a42 100644 --- a/mmv1/products/dataplex/Task.yaml +++ b/mmv1/products/dataplex/Task.yaml @@ -17,7 +17,7 @@ self_link: 'projects/{{project}}/locations/{{location}}/lakes/{{lake}}/tasks/{{t create_url: 'projects/{{project}}/locations/{{location}}/lakes/{{lake}}/tasks?task_id={{task_id}}' update_url: 'projects/{{project}}/locations/{{location}}/lakes/{{lake}}/tasks/{{task_id}}' delete_url: 'projects/{{project}}/locations/{{location}}/lakes/{{lake}}/tasks/{{task_id}}' -update_verb: :GET +update_verb: :POST update_mask: true description: | A Dataplex task represents the work that you want Dataplex to do on a schedule. It encapsulates code, parameters, and the schedule. @@ -51,7 +51,7 @@ iam_policy: !ruby/object:Api::Resource::IamPolicy exclude: false method_name_separator: ':' parent_resource_attribute: 'task_id' - fetch_iam_policy_verb: :POST + fetch_iam_policy_verb: :GET import_format: [ 'projects/{{project}}/locations/{{location}}/lakes/{{lake}}/tasks/{{task_id}}', diff --git a/mmv1/templates/terraform/examples/dataplex_task_notebook.tf.erb b/mmv1/templates/terraform/examples/dataplex_task_notebook.tf.erb index d90461079c62..cbf395520186 100644 --- a/mmv1/templates/terraform/examples/dataplex_task_notebook.tf.erb +++ b/mmv1/templates/terraform/examples/dataplex_task_notebook.tf.erb @@ -33,7 +33,7 @@ resource "google_dataplex_task" "<%= ctx[:primary_resource_id] %>" { } } notebook { - notebook = "gs://terraform-test4/test-notebook.ipynb" + notebook = "gs://terraform-test/test-notebook.ipynb" infrastructure_spec { batch { executors_count = 2 From 8067f540c780242f292150c7d79473d3daa2c10e Mon Sep 17 00:00:00 2001 From: sachitjaggi156 Date: Thu, 13 Jul 2023 21:40:38 +0000 Subject: [PATCH 19/19] Corrected Update Verb --- mmv1/products/dataplex/Task.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmv1/products/dataplex/Task.yaml b/mmv1/products/dataplex/Task.yaml index 2abf68fb5a42..3a4095794264 100644 --- a/mmv1/products/dataplex/Task.yaml +++ b/mmv1/products/dataplex/Task.yaml @@ -17,7 +17,7 @@ self_link: 'projects/{{project}}/locations/{{location}}/lakes/{{lake}}/tasks/{{t create_url: 'projects/{{project}}/locations/{{location}}/lakes/{{lake}}/tasks?task_id={{task_id}}' update_url: 'projects/{{project}}/locations/{{location}}/lakes/{{lake}}/tasks/{{task_id}}' delete_url: 'projects/{{project}}/locations/{{location}}/lakes/{{lake}}/tasks/{{task_id}}' -update_verb: :POST +update_verb: :PATCH update_mask: true description: | A Dataplex task represents the work that you want Dataplex to do on a schedule. It encapsulates code, parameters, and the schedule.