this is where I found it:
opened 12:17PM - 10 Jun 22 UTC
closed 02:29PM - 30 Jul 22 UTC
question
## Terraform version, Kubernetes provider version and Kubernetes version
```
T… erraform version: 1.2.1
Go runtime version: go1.18.1
hashicorp/kubernetes/2.11.0
kubectl Version:"v1.24.1"
```
## Terraform configuration
A lot is missing but you should get the idea.
```hcl
### GKE Module
resource "google_container_cluster" "primary" {
provider = google-beta
name = var.cluster_name
location = var.location
project = var.project
remove_default_node_pool = true
initial_node_count = 1
....
}
output "cluster_ca_certificate" {
value = google_container_cluster.primary.master_auth.0.cluster_ca_certificate
}
output "endpoint" {
value = google_container_cluster.primary.endpoint
}
### Flux module
data "google_client_config" "default" {}
provider "kubernetes" {
host = "https://${module.gke.endpoint}"
token = data.google_client_config.default.access_token
cluster_ca_certificate = base64decode(
module.gke.cluster_ca_certificate,
)
}
locals {
raw_emissary_manifests = split("---", file("${path.root}/flux-config/conditionals/${var.env_type}/ambassador.yaml"))
hcl_emissary_manifests = [for manifest in local.raw_emissary_manifests : yamldecode(manifest)]
emissary_cfg = base64encode(<<YAML
service:
annotations:
cloud.google.com/load-balancer-type: "${(var.external) ? "External" : "Internal"}"
external-dns.alpha.kubernetes.io/hostname: ${(var.external) ? "amb.${var.cluster_name}.bestsellerit.com" : "ambassador.${var.cluster_name}.k8s.bestcorp.net"}
tags.datadoghq.com/env: "${var.env_type}"
podLabels:
tags.datadoghq.com/env: "${var.env_type}"
YAML
)
}
resource "kubernetes_cluster_role_binding" "admin" {
metadata {
name = "cluster-admin-binding"
}
role_ref {
api_group = "rbac.authorization.k8s.io"
kind = "ClusterRole"
name = "cluster-admin"
}
subject {
kind = "User"
name = var.deployment_account_email
api_group = "rbac.authorization.k8s.io"
}
}
[...]
resource "kubernetes_manifest" "emissary_ns" {
depends_on = [kubernetes_cluster_role_binding.admin, helm_release.gatekeeper]
computed_fields = ["metadata.labels", "metadata.annotations","spec.finalizers","status"]
manifest = yamldecode(file("${path.module}/conditionals/emissary/namespace.yaml"))
}
resource "kubernetes_manifest" "emissary_cert" {
count = var.ambassador ? 1 : 0
depends_on = [kubectl_manifest.sync_flux, kubernetes_manifest.emissary_ns]
computed_fields = ["metadata.labels", "metadata.annotations","spec.finalizers","status"]
manifest = yamldecode(templatefile("${path.module}/conditionals/emissary/certificate.yaml", {
dns = (var.external) ? "amb.${var.cluster_name}.bestsellerit.com" : "ambassador.${var.cluster_name}.k8s.bestcorp.net"
}))
}
resource "kubernetes_manifest" "emissary_cfg" {
count = var.ambassador ? 1 : 0
depends_on = [kubectl_manifest.sync_flux, kubernetes_manifest.emissary_ns]
computed_fields = ["metadata.labels", "metadata.annotations","spec.finalizers","status"]
manifest = yamldecode(templatefile("${path.module}/conditionals/emissary/helm-config.yaml", { dataBase64 : local.emissary_cfg }))
}
```
## Question
```
Hi, i have some kubernetes resources that i was managing using the old kubectl provider. I have removed them from the state of the old provider and imported them into the new one.
I have two problems:
1. Terraform wants to destroy my imported resources and there is no prompt stating what the reason is, this is not so important right now and maybe it's an improvement for a future version.
2. I am getting a cycle error on destroy. I am not sure why the gke part depends on the kubectl_manifest
Error: Cycle: module.flux.kubernetes_manifest.emissary_docs[0] (destroy), module.flux.kubernetes_manifest.emissary_docs[3] (destroy), module.flux.kubernetes_manifest.emissary_docs[1] (destroy), module.gke.output.cluster_ca_certificate (expand), module.gke.output.endpoint (expand), provider["registry.terraform.io/hashicorp/kubernetes"], module.flux.kubernetes_manifest.emissary_docs[2] (destroy), module.gke.google_container_cluster.primary
```
Debug output:
```
2022-06-10T06:13:31.649Z [ERROR] Graph validation failed. Graph:
[...]
module.gke.output.cluster_ca_certificate (expand)
module.gke (expand)
module.gke.google_container_cluster.primary
module.gke.google_container_cluster.primary (expand)
[...]
module.gke.output.endpoint (expand)
module.gke (expand)
module.gke.google_container_cluster.primary
module.gke.google_container_cluster.primary (expand)
[...]
provider["registry.terraform.io/hashicorp/kubernetes"]
module.gke.output.cluster_ca_certificate (expand)
module.gke.output.endpoint (expand)
[...]
module.flux.kubernetes_manifest.emissary_docs[0] (destroy)
module.vault.null_resource.create_cluster_policies (destroy)
provider["registry.terraform.io/hashicorp/kubernetes"]
module.flux.kubernetes_manifest.emissary_docs[1] (destroy)
module.vault.null_resource.create_cluster_policies (destroy)
provider["registry.terraform.io/hashicorp/kubernetes"]
[...]
module.flux.kubernetes_manifest.emissary_docs[2] (destroy)
module.vault.null_resource.create_cluster_policies (destroy)
provider["registry.terraform.io/hashicorp/kubernetes"]
[...]
module.flux.kubernetes_manifest.emissary_docs[3] (destroy)
module.vault.null_resource.create_cluster_policies (destroy)
provider["registry.terraform.io/hashicorp/kubernetes"]
[...]
module.gke.google_container_cluster.primary
module.flux.kubernetes_manifest.emissary_docs[0] (destroy)
module.flux.kubernetes_manifest.emissary_docs[1] (destroy)
module.flux.kubernetes_manifest.emissary_docs[2] (destroy)
module.flux.kubernetes_manifest.emissary_docs[3] (destroy)
module.flux.time_sleep.wait_60_seconds (destroy)
module.gke (expand)
module.gke.data.google_container_engine_versions.k8s_version (expand)
module.gke.data.google_project.project (expand)
module.gke.google_bigquery_dataset.dataset (expand)
module.gke.google_container_cluster.primary (expand)
module.gke.local.network (expand)
module.gke.local.subnetwork (expand)
module.gke.var.cluster_name (expand)
module.gke.var.cluster_secondary_range_name (expand)
module.gke.var.enable_gke_ingress (expand)
module.gke.var.external (expand)
module.gke.var.external_auto_subnet (expand)
module.gke.var.location (expand)
module.gke.var.master_ipv4_cidr_block (expand)
module.gke.var.namespaces (expand)
module.gke.var.project (expand)
module.gke.var.services_secondary_range_name (expand)
module.vault.null_resource.create_cluster_policies (destroy)
provider["registry.terraform.io/hashicorp/google-beta"]
```