Terraform sync tool

tools/terraform_sync_tool/cloudbuild.yaml

@@ -0,0 +1,14 @@
+steps:
+  # step 0: run terraform commands in deploy.sh to detects drifts
+  - name: 'alpine/terragrunt'
+    entrypoint: 'bash'
+    dir: './tools/terraform_sync_tool/'
+    args: ['deploy.sh', 'qa', 'terraform-sync-tool']
+
+  # step 1: run python scripts to investigate terraform output
+  - name: python:3.7
+    entrypoint: 'bash'
+    dir: './tools/terraform_sync_tool/'
+    args: 
+    - -c
+    - 'pip install -r ./requirements.txt && python terraform_sync.py'

tools/terraform_sync_tool/deploy.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+env=$1
+tool=$2
+
+terragrunt run-all plan -json --terragrunt-non-interactive --terragrunt-working-dir="${env}"/"${tool}" > state.json
+# terragrunt run-all plan -json --terragrunt-non-interactive --terragrunt-working-dir=/qa/terraform-sync-tool > state.json

tools/terraform_sync_tool/modules/terraform-sync-tool/main.tf

@@ -0,0 +1,96 @@
+locals {
+  datasets        = { for dataset in var.datasets : dataset["dataset_id"] => dataset }
+  tables          = { for table in var.tables : table["table_id"] => table }
+  views           = { for view in var.views : view["view_id"] => view }
+
+  iam_to_primitive = {
+    "roles/bigquery.dataOwner" : "OWNER"
+    "roles/bigquery.dataEditor" : "WRITER"
+    "roles/bigquery.dataViewer" : "READER"
+  }
+}
+
+#this is the test for dataset list creation
+resource "google_bigquery_dataset" "bq_dataset" {
+  for_each        = local.datasets
+  friendly_name   = each.value["friendly_name"]
+  dataset_id      = each.key
+  location        = each.value["location"] 
+  project         = var.project_id
+
+  dynamic "default_encryption_configuration" {
+    for_each = var.encryption_key == null ? [] : [var.encryption_key]
+    content {
+      kms_key_name = var.encryption_key
+    }
+  }
+
+  dynamic "access" {
+    for_each = var.access
+
+    content {
+      # BigQuery API converts IAM to primitive roles in its backend.
+      # This causes Terraform to show a diff on every plan that uses IAM equivalent roles.
+      # Thus, do the conversion between IAM to primitive role here to prevent the diff.
+      role = lookup(local.iam_to_primitive, access.value.role, access.value.role)
+
+      domain         = lookup(access.value, "domain", null)
+      group_by_email = lookup(access.value, "group_by_email", null)
+      user_by_email  = lookup(access.value, "user_by_email", null)
+      special_group  = lookup(access.value, "special_group", null)
+    }
+  }
+ }
+
+resource "google_bigquery_table" "bq_table" {
+  for_each        = local.tables
+  dataset_id      = each.value["dataset_id"]
+  friendly_name   = each.key
+  table_id        = each.key
+  labels          = each.value["labels"]
+  schema          = file(each.value["schema"])
+  clustering      = each.value["clustering"]
+  expiration_time = each.value["expiration_time"]
+  project         = var.project_id
+  deletion_protection = each.value["deletion_protection"]
+  depends_on = [google_bigquery_dataset.bq_dataset]
+
+  dynamic "time_partitioning" {
+    for_each = each.value["time_partitioning"] != null ? [each.value["time_partitioning"]] : []
+    content {
+      type                     = time_partitioning.value["type"]
+      expiration_ms            = time_partitioning.value["expiration_ms"]
+      field                    = time_partitioning.value["field"]
+      require_partition_filter = time_partitioning.value["require_partition_filter"]
+    }
+  }
+
+  dynamic "range_partitioning" {
+    for_each = each.value["range_partitioning"] != null ? [each.value["range_partitioning"]] : []
+    content {
+      field = range_partitioning.value["field"]
+      range {
+        start    = range_partitioning.value["range"].start
+        end      = range_partitioning.value["range"].end
+        interval = range_partitioning.value["range"].interval
+      }
+    }
+  }
+
+}
+
+resource "google_bigquery_table" "bq_view" {
+  for_each      = local.views
+  dataset_id    = each.value["dataset_id"]
+  friendly_name = each.key
+  table_id      = each.key
+  labels        = each.value["labels"]
+  project       = var.project_id
+  deletion_protection = each.value["deletion_protection"]
+  depends_on = [google_bigquery_table.bq_table]
+
+  view {
+    query          = each.value["query"]
+    use_legacy_sql = each.value["use_legacy_sql"]
+  }
+}

tools/terraform_sync_tool/modules/terraform-sync-tool/variables.tf

@@ -0,0 +1,113 @@
+variable "description" {
+  description = "Dataset description."
+  type        = string
+  default     = null
+}
+
+variable "location" {
+  description = "The regional location for the dataset only US and EU are allowed in module"
+  type        = string
+  default     = "US"
+}
+
+variable "delete_contents_on_destroy" {
+  description = "(Optional) If set to true, delete all the tables in the dataset when destroying the resource; otherwise, destroying the resource will fail if tables are present."
+  type        = bool
+  default     = null
+}
+
+variable "deletion_protection" {
+  description = "Whether or not to allow Terraform to destroy the instance. Unless this field is set to false in Terraform state, a terraform destroy or terraform apply that would delete the instance will fail."
+  type        = bool
+  default     = true
+  }
+
+variable "default_table_expiration_ms" {
+  description = "TTL of tables using the dataset in MS"
+  type        = number
+  default     = null
+}
+
+variable "project_id" {
+  description = "Project where the dataset and table are created"
+  type        = string
+}
+
+variable "encryption_key" {
+  description = "Default encryption key to apply to the dataset. Defaults to null (Google-managed)."
+  type        = string
+  default     = null
+}
+
+variable "dataset_labels" {
+  description = "Key value pairs in a map for dataset labels"
+  type        = map(string)
+  default     = {}
+}
+
+# Format: list(objects)
+# domain: A domain to grant access to.
+# group_by_email: An email address of a Google Group to grant access to.
+# user_by_email:  An email address of a user to grant access to.
+# special_group: A special group to grant access to.
+
+variable "access" {
+  description = "An array of objects that define dataset access for one or more entities."
+  type        = any
+
+  # At least one owner access is required.
+  default = [{
+    role          = "roles/bigquery.dataOwner"
+    special_group = "projectOwners"
+  }]
+}
+variable "datasets" {
+  description = "this is a test DS"
+  default = []
+  type = list(object({
+        dataset_id = string
+        friendly_name = string
+        location = string
+   }
+  ))
+}
+variable "tables" {
+  description = "A list of objects which include table_id, schema, clustering, time_partitioning, expiration_time and labels."
+  default     = []
+  type = list(object({
+    table_id   = string,
+    dataset_id = string, #added to test creating multi dataset
+    schema     = string,
+    clustering = list(string),
+    deletion_protection=bool,
+    time_partitioning = object({
+      expiration_ms            = string,
+      field                    = string,
+      type                     = string,
+      require_partition_filter = bool,
+    }),
+	range_partitioning = object({
+      field = string,
+      range = object({
+        start    = string,
+        end      = string,
+        interval = string,
+      }),
+    }),
+    expiration_time = string,
+    labels          = map(string),
+  }
+  ))
+}
+variable "views" {
+  description = "A list of objects which include table_id, which is view id, and view query"
+  default     = []
+  type = list(object({
+    view_id        = string,
+    dataset_id     = string,
+    query          = string,
+    deletion_protection=bool,
+    use_legacy_sql = bool,
+    labels         = map(string),
+  }))
+}

tools/terraform_sync_tool/qa/terraform-sync-tool/json_schemas/TableForTest.json

@@ -0,0 +1,56 @@
+  [
+    {
+      "description": "Col1",
+      "mode": "NULLABLE",
+      "name": "Col1",
+      "type": "STRING"
+    },
+    {
+      "description": "Col2",
+      "mode": "NULLABLE",
+      "name": "Col2",
+      "type": "STRING"
+    },
+    {
+      "description": "Col3",
+      "mode": "NULLABLE",
+      "name": "Col3",
+      "type": "STRING"
+    },
+    {
+      "description": "Col4",
+      "mode": "NULLABLE",
+      "name": "Col4",
+      "type": "STRING"
+    },
+    {
+      "description": "Col5",
+      "mode": "NULLABLE",
+      "name": "Col5",
+      "type": "STRING"
+    },
+    {
+      "description": "Col6",
+      "mode": "NULLABLE",
+      "name": "Col6",
+      "type": "STRING"
+    },
+    {
+      "description": "Col7",
+      "mode": "NULLABLE",
+      "name": "Col7",
+      "type": "STRING"
+    },
+    {
+      "description": "Col8",
+      "mode": "NULLABLE",
+      "name": "Col8",
+      "type": "STRING"
+    },
+    {
+      "description": "Col9",
+      "mode": "NULLABLE",
+      "name": "Col9",
+      "type": "STRING"
+    }
+  ] 

tools/terraform_sync_tool/qa/terraform-sync-tool/terragrunt.hcl

@@ -0,0 +1,39 @@
+terraform {
+  source = "../../modules/terraform-sync-tool"
+}
+
+include "root" {
+  path   = find_in_parent_folders()
+  expose = true
+}
+
+locals {
+  dataset_id = "tf_test_sync_tool"
+}
+
+inputs = {
+  project_id = include.root.inputs.project_id
+  # The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
+  datasets = [
+    {
+      dataset_id    = "${local.dataset_id}"
+      friendly_name = "Dataset for Terraform Sync Tool"
+      location      = "US"
+      labels        = {}
+    }
+  ]
+
+  tables = [
+    {
+      table_id            = "TableForTest"
+      dataset_id          = "${local.dataset_id}"
+      schema              = "json_schemas/TableForTest.json"
+      clustering          = []
+      expiration_time     = null
+      deletion_protection = true
+      range_partitioning  = null
+      time_partitioning   = null
+      labels              = {}
+    }
+  ]
+}

tools/terraform_sync_tool/qa/terragrunt.hcl

@@ -0,0 +1,42 @@
+# Indicate where to source the terraform module from.
+# The URL used here is a shorthand for
+# "tfr://registry.terraform.io/terraform-aws-modules/vpc/aws?version=3.5.0".
+# Note the extra `/` after the protocol is required for the shorthand
+# notation.
+
+locals {
+  gcp_project_id = "candicehou-terraform-sync-tool"
+}
+
+inputs = {
+  project_id = local.gcp_project_id
+  gcp_region = "us-central1"
+}
+
+generate "provider" {
+  path      = "provider.tf"
+  if_exists = "overwrite"
+  contents  = <<EOF
+provider "google" {
+  project = "${local.gcp_project_id}"
+}
+  EOF
+}
+
+remote_state {
+  backend = "gcs"
+  config = {
+    project  = local.gcp_project_id
+    location = "us"
+    bucket   = "synctooltest"
+    prefix   = "qa/${path_relative_to_include()}/terraform.tfstate"
+    gcs_bucket_labels = {
+      owner = "terragrunt_test"
+      name  = "terraform_state_storage"
+    }
+  }
+  generate = {
+    path      = "backend.tf"
+    if_exists = "overwrite_terragrunt"
+  }
+}

tools/terraform_sync_tool/requirements.txt

@@ -0,0 +1 @@
+google-cloud-bigquery

tools/terraform_sync_tool/state.json

@@ -0,0 +1,6 @@
+{"@level":"info","@message":"Terraform 1.1.9","@module":"terraform.ui","@timestamp":"2022-06-08T13:22:12.564592-04:00","terraform":"1.1.9","type":"version","ui":"1.0"}
+{"@level":"info","@message":"google_bigquery_dataset.bq_dataset[\"tf_test_sync_tool\"]: Refreshing state... [id=projects/candicehou-terraform-sync-tool/datasets/tf_test_sync_tool]","@module":"terraform.ui","@timestamp":"2022-06-08T13:22:14.589057-04:00","hook":{"resource":{"addr":"google_bigquery_dataset.bq_dataset[\"tf_test_sync_tool\"]","module":"","resource":"google_bigquery_dataset.bq_dataset[\"tf_test_sync_tool\"]","implied_provider":"google","resource_type":"google_bigquery_dataset","resource_name":"bq_dataset","resource_key":"tf_test_sync_tool"},"id_key":"id","id_value":"projects/candicehou-terraform-sync-tool/datasets/tf_test_sync_tool"},"type":"refresh_start"}
+{"@level":"info","@message":"google_bigquery_dataset.bq_dataset[\"tf_test_sync_tool\"]: Refresh complete [id=projects/candicehou-terraform-sync-tool/datasets/tf_test_sync_tool]","@module":"terraform.ui","@timestamp":"2022-06-08T13:22:15.112837-04:00","hook":{"resource":{"addr":"google_bigquery_dataset.bq_dataset[\"tf_test_sync_tool\"]","module":"","resource":"google_bigquery_dataset.bq_dataset[\"tf_test_sync_tool\"]","implied_provider":"google","resource_type":"google_bigquery_dataset","resource_name":"bq_dataset","resource_key":"tf_test_sync_tool"},"id_key":"id","id_value":"projects/candicehou-terraform-sync-tool/datasets/tf_test_sync_tool"},"type":"refresh_complete"}
+{"@level":"info","@message":"google_bigquery_table.bq_table[\"TableForTest\"]: Refreshing state... [id=projects/candicehou-terraform-sync-tool/datasets/tf_test_sync_tool/tables/TableForTest]","@module":"terraform.ui","@timestamp":"2022-06-08T13:22:15.119610-04:00","hook":{"resource":{"addr":"google_bigquery_table.bq_table[\"TableForTest\"]","module":"","resource":"google_bigquery_table.bq_table[\"TableForTest\"]","implied_provider":"google","resource_type":"google_bigquery_table","resource_name":"bq_table","resource_key":"TableForTest"},"id_key":"id","id_value":"projects/candicehou-terraform-sync-tool/datasets/tf_test_sync_tool/tables/TableForTest"},"type":"refresh_start"}
+{"@level":"info","@message":"google_bigquery_table.bq_table[\"TableForTest\"]: Refresh complete [id=projects/candicehou-terraform-sync-tool/datasets/tf_test_sync_tool/tables/TableForTest]","@module":"terraform.ui","@timestamp":"2022-06-08T13:22:15.275101-04:00","hook":{"resource":{"addr":"google_bigquery_table.bq_table[\"TableForTest\"]","module":"","resource":"google_bigquery_table.bq_table[\"TableForTest\"]","implied_provider":"google","resource_type":"google_bigquery_table","resource_name":"bq_table","resource_key":"TableForTest"},"id_key":"id","id_value":"projects/candicehou-terraform-sync-tool/datasets/tf_test_sync_tool/tables/TableForTest"},"type":"refresh_complete"}
+{"@level":"info","@message":"Plan: 0 to add, 0 to change, 0 to destroy.","@module":"terraform.ui","@timestamp":"2022-06-08T13:22:15.289118-04:00","changes":{"add":0,"change":0,"remove":0,"operation":"plan"},"type":"change_summary"}