The Last of the Facts Modules

.gitignore

@@ -11,7 +11,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Temp files
+*~
+*#
 *.swp
+
 coverage/*
 *.pyc
 
@@ -21,3 +25,13 @@ coverage/*
 
 # Keep project coverage statistics under source control
 !coverage/.last_run.json
+
+# OS generated files
+.DS_Store
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes
+ehthumbs.db
+Thumbs.db
+

.travis.yml

@@ -1,8 +1,7 @@
 language: ruby # ruby version defined in .ruby-version will be used
 
 script:
-- bundle exec rake test:rubocop
-- bundle exec rake test:spec
+- bundle exec rake test
 
 git:
   submodules: false

.yamllint

@@ -0,0 +1,8 @@
+extends: default
+
+rules:
+  line-length: disable
+  # Comments indentation will look weird
+  # because we want indentation to look
+  # proper in unindentented versions
+  comments: disable

Rakefile

@@ -11,19 +11,130 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Configuration
 $LOAD_PATH.unshift File.dirname(__FILE__)
 Dir.chdir(File.dirname(__FILE__))
 
-# Load all tasks from tasks/
-Dir[File.join('tasks', '*.rb')].reject { |p| File.directory? p }
-                               .each do |f|
-  require f
+PROVIDER_FOLDERS = {
+  ansible: 'build/ansible',
+  puppet: 'build/puppet/%<mod>s',
+  chef: 'build/chef/%<mod>s',
+  terraform: 'build/terraform'
+}.freeze
+
+# Requires
+require 'rspec/core/rake_task'
+require 'rubocop/rake_task'
+require 'tempfile'
+
+# Requires for YAML linting.
+require 'api/async'
+require 'api/bundle'
+require 'api/product'
+require 'api/resource'
+require 'api/type'
+require 'compile/core'
+require 'google/yaml_validator'
+
+RSpec::Core::RakeTask.new(:spec)
+RuboCop::RakeTask.new
+
+# YAML Linting
+# This class calls our provider code to get the printed contents of the
+# compiled YAML. We run the linter on this printed version (so, no embedded
+# ERB)
+class YamlLinter
+  include Compile::Core
+
+  def yaml_contents(file)
+    source = compile(file)
+    config = Google::YamlValidator.parse(source)
+    unless config.class <= Api::Product
+      raise StandardError, "#{file} is #{config.class}"\
+        ' instead of Api::Product' \
+    end
+    # Compile step #2: Now that we have the target class, compile with that
+    # class features
+    config.compile(file, 0)
+  end
+end
+
+# Handles finding the list of products for a given provider.
+class Providers
+  def self.provider_list
+    PROVIDER_FOLDERS.keys
+  end
+
+  # All possible products that exist.
+  def self.all_products
+    products = File.join(File.dirname(__FILE__), 'products')
+    Dir.glob("#{products}/**/api.yaml")
+  end
+
+  def initialize(name)
+    @name = name
+  end
+
+  def products
+    products = File.join(File.dirname(__FILE__), 'products')
+    files = Dir.glob("#{products}/**/#{@name}.yaml")
+    files.map do |file|
+      match = file.match(%r{^.*products\/([_a-z]*)\/.*yaml.*})
+      match&.captures&.at(0)
+    end.compact
+  end
+
+  def compile_module(mod)
+    folder = format(PROVIDER_FOLDERS[@name.to_sym], mod: mod)
+    flag = "COMPILER_#{folder.gsub('build/', '').tr('/', '_').upcase}_OUTPUT"
+    output = ENV[flag] || format(PROVIDER_FOLDERS[@name.to_sym], mod: mod)
+    %x(bundle exec compiler -p products/#{mod} -e #{@name} -o #{output})
+  end
+
+  def compilation_targets
+    products.map { |prod| "compile:#{@name}:#{prod}" }
+  end
 end
 
-# Find all tasks under the test namespace
-# Ignore those with multiple levels like rubocop:auto_correct
-tests = Rake.application.tasks.select do |task|
-  /^test:[a-z]*$/ =~ task.name
-end.map(&:name)
+# Test Tasks
+desc 'Run all of the MM tests (rubocop, rspec)'
+multitask test: %w[rubocop spec]
+
+desc 'Lints all of the compiled YAML files'
+task :yamllint do
+  Providers.all_products.each do |file|
+    tempfile = Tempfile.new
+    tempfile.write(YamlLinter.new.yaml_contents(file))
+    tempfile.rewind
+    puts %x(yamllint -c #{File.join(File.dirname(__FILE__), '.yamllint')} #{tempfile.path})
+    tempfile.close
+    tempfile.unlink
+  end
+end
 
-multitask 'test' => tests
+# Compiling Tasks
+compile_list = Providers.provider_list.map do |x|
+  Providers.new(x).compilation_targets
+end.flatten
+
+desc 'Compile all modules'
+multitask compile: compile_list
+
+namespace :compile do
+  Providers.provider_list.each do |provider|
+    # Each provider should default to compiling everything.
+    desc "Compile all modules for #{provider.capitalize}"
+    prov = Providers.new(provider)
+    multitask provider.to_sym => prov.compilation_targets
+
+    namespace provider.to_sym do
+      prov.products.each do |mod|
+        # Each module should have its own task for compiling.
+        desc "Compile the #{mod} module for #{provider.capitalize}"
+        task mod.to_sym do
+          prov.compile_module(mod)
+        end
+      end
+    end
+  end
+end

products/compute/ansible.yaml

@@ -269,6 +269,7 @@ overrides: !ruby/object:Provider::ResourceOverrides
       - 'products/compute/helpers/python/instance_metadata.py'
   InterconnectAttachment: !ruby/object:Provider::Ansible::ResourceOverride
     version_added: '2.8'
+    has_tests: false
   Network: !ruby/object:Provider::Ansible::ResourceOverride
     properties:
       routingConfig: !ruby/object:Provider::Ansible::PropertyOverride

products/container/ansible.yaml

@@ -23,7 +23,19 @@ manifest: !ruby/object:Provider::Ansible::Manifest
     - google-auth >= 1.3.0
   version_added: '2.6'
   author: Google Inc. (@googlecloudplatform)
-# This is where custom code would be defined eventually.
+datasources: !ruby/object:Provider::ResourceOverrides
+  Cluster: !ruby/object:Provider::Ansible::ResourceOverride
+    version_added: '2.8'
+    facts: !ruby/object:Provider::Ansible::FactsOverride
+      has_filters: false
+      list_key: 'clusters'
+  NodePool: !ruby/object:Provider::Ansible::ResourceOverride
+    version_added: '2.8'
+    facts: !ruby/object:Provider::Ansible::FactsOverride
+      has_filters: false
+      list_key: 'nodePools'
+  KubeConfig: !ruby/object:Provider::Ansible::ResourceOverride
+    exclude: true
 overrides: !ruby/object:Provider::ResourceOverrides
   Cluster: !ruby/object:Provider::Ansible::ResourceOverride
     provider_helpers:

products/container/examples/ansible/cluster.yaml

@@ -14,7 +14,7 @@
 task: !ruby/object:Provider::Ansible::Task
   name: gcp_container_cluster
   code:
-    name: <%= ctx[:name] %>
+    name: 'my-cluster'
     initial_node_count: 2
     master_auth:
       username: "cluster_admin"
@@ -26,12 +26,3 @@ task: !ruby/object:Provider::Ansible::Task
     project: <%= ctx[:project] %>
     auth_kind: <%= ctx[:auth_kind] %>
     service_account_file: <%= ctx[:service_account_file] %>
-verifier: !ruby/object:Provider::Ansible::Verifier
-  command: |
-    gcloud container clusters describe
-      --project="{{ gcp_project}}"
-      --zone=us-central1-a
-      "{{ resource_name }}"
-  failure: !ruby/object:Provider::Ansible::FailureCondition
-    error: |
-      No cluster named '{{ resource_name }}' in {{ gcp_project }}.

products/container/examples/ansible/node_pool.yaml

@@ -15,7 +15,7 @@
 task: !ruby/object:Provider::Ansible::Task
   name: gcp_container_node_pool
   code:
-      name: <%= ctx[:name] %>
+      name: 'my-pool'
       initial_node_count: 4
       cluster: "{{ cluster }}"
       zone: 'us-central1-a'

products/spanner/ansible.yaml

@@ -23,6 +23,20 @@ manifest: !ruby/object:Provider::Ansible::Manifest
     - google-auth >= 1.3.0
   version_added: '2.7'
   author: Google Inc. (@googlecloudplatform)
+datasources: !ruby/object:Provider::ResourceOverrides
+  Instance: !ruby/object:Provider::Ansible::ResourceOverride
+    version_added: '2.8'
+    facts: !ruby/object:Provider::Ansible::FactsOverride
+      has_filters: false
+      list_key: 'instances'
+  Database: !ruby/object:Provider::Ansible::ResourceOverride
+    version_added: '2.8'
+    facts: !ruby/object:Provider::Ansible::FactsOverride
+      has_filters: false
+      list_key: 'databases'
+  # Virtual => true objects that don't need dedicated resources.
+  InstanceConfig: !ruby/object:Provider::Ansible::ResourceOverride
+    exclude: true
 overrides: !ruby/object:Provider::ResourceOverrides
   Instance: !ruby/object:Provider::Ansible::ResourceOverride
     custom_create_resource: true

products/spanner/examples/ansible/database.yaml

@@ -33,13 +33,3 @@ task: !ruby/object:Provider::Ansible::Task
       project: <%= ctx[:project] %>
       auth_kind: <%= ctx[:auth_kind] %>
       service_account_file: <%= ctx[:service_account_file] %>
-verifier: !ruby/object:Provider::Ansible::Verifier
-  command: |
-    gcloud spanner databases describe
-      --project="{{ gcp_project }}"
-      --instance="{{ instance.name }}"
-      "webstore"
-  failure: !ruby/object:Provider::Ansible::GrpcFailureCondition
-    single: Database
-    plural: instances/instance-database/databases
-    name: webstore

products/spanner/examples/ansible/instance.yaml

@@ -23,11 +23,3 @@ task: !ruby/object:Provider::Ansible::Task
     project: <%= ctx[:project] %>
     auth_kind: <%= ctx[:auth_kind] %>
     service_account_file: <%= ctx[:service_account_file] %>
-verifier: !ruby/object:Provider::Ansible::Verifier
-  command: |
-    gcloud spanner instances describe
-      --project="{{ gcp_project }}"
-      "{{ resource_name }}"
-  failure: !ruby/object:Provider::Ansible::GrpcFailureCondition
-    single: Instance
-    plural: instances

products/sql/ansible.yaml

@@ -23,7 +23,25 @@ manifest: !ruby/object:Provider::Ansible::Manifest
     - google-auth >= 1.3.0
   version_added: '2.7'
   author: Google Inc. (@googlecloudplatform)
-# This is where custom code would be defined eventually.
+datasources: !ruby/object:Provider::ResourceOverrides
+  Instance: !ruby/object:Provider::Ansible::ResourceOverride
+    version_added: '2.8'
+    facts: !ruby/object:Provider::Ansible::FactsOverride
+      has_filters: false
+  Database: !ruby/object:Provider::Ansible::ResourceOverride
+    version_added: '2.8'
+    facts: !ruby/object:Provider::Ansible::FactsOverride
+      has_filters: false
+  User: !ruby/object:Provider::Ansible::ResourceOverride
+    version_added: '2.8'
+    facts: !ruby/object:Provider::Ansible::FactsOverride
+      has_filters: false
+  Tier: !ruby/object:Provider::Ansible::ResourceOverride
+    exclude: true
+  SslCert: !ruby/object:Provider::Ansible::ResourceOverride
+    exclude: true
+  Flag: !ruby/object:Provider::Ansible::ResourceOverride
+    exclude: true
 overrides: !ruby/object:Provider::ResourceOverrides
   Instance: !ruby/object:Provider::Ansible::ResourceOverride
     access_api_results: true

products/sql/examples/ansible/database.yaml

@@ -36,11 +36,3 @@ task: !ruby/object:Provider::Ansible::Task
     project: <%= ctx[:project] %>
     auth_kind: <%= ctx[:auth_kind] %>
     service_account_file: <%= ctx[:service_account_file] %>
-verifier: !ruby/object:Provider::Ansible::Verifier
-  command: |
-    gcloud sql databases describe
-      --instance='instance-database'
-      --project="{{ gcp_project}}"
-      "{{ resource_name }}"
-  failure: !ruby/object:Provider::Ansible::FailureCondition
-    error: '(gcloud.sql.databases.describe) HTTPError 404: Not Found'

products/sql/examples/ansible/instance.yaml

@@ -25,10 +25,3 @@ task: !ruby/object:Provider::Ansible::Task
     project: <%= ctx[:project] %>
     auth_kind: <%= ctx[:auth_kind] %>
     service_account_file: <%= ctx[:service_account_file] %>
-verifier: !ruby/object:Provider::Ansible::Verifier
-  command: |
-    gcloud sql instances describe
-      --project="{{ gcp_project}}"
-      "{{ resource_name }}"
-  failure: !ruby/object:Provider::Ansible::FailureCondition
-    error: Cloud SQL instance does not exist

provider/ansible.rb

@@ -244,6 +244,7 @@ def generate_resource_tests(data)
         path = ["products/#{data[:product_name]}",
                 "examples/ansible/#{prod_name}.yaml"].join('/')
 
+        return unless data[:object].has_tests
         # Unlike other providers, all resources will not be built at once or
         # in close timing to each other (due to external PRs).
         # This means that examples might not be built out for every resource

provider/ansible/resource_override.rb

@@ -25,6 +25,7 @@ module OverrideProperties
       attr_reader :create
       attr_reader :delete
       attr_reader :editable
+      attr_reader :has_tests
       attr_reader :hidden
       attr_reader :imports
       attr_reader :post_create
@@ -51,6 +52,7 @@ def validate
         default_value_property :custom_update_resource, false
         default_value_property :exclude, false
         default_value_property :editable, true
+        default_value_property :has_tests, true
         default_value_property :imports, []
         default_value_property :provider_helpers, []
         default_value_property :unwrap_resource, false
@@ -62,6 +64,7 @@ def validate
         check_optional_property :create, ::String
         check_optional_property :delete, ::String
         check_property :editable, :boolean
+        check_property :has_tests, :boolean
         check_optional_property :hidden, ::Array
         check_property :imports, ::Array
         check_optional_property :post_create, ::String