terraform输出Google Kubernetes群集Ingress负载均衡器ip

我已经成功地用terraform自动化了kubernetes集群部署。启动群集后，terraform还使用资源调配（使用本地exec运行.sh脚本）将我的应用部署到群集。我还将入口添加到集群中，我需要在入口负载平衡器创建后获得它的IP。更好的选择是地形输出。 我现在得到它的方式是在脚本末尾运行这部分代码

IP="$(kubectl get ingress appname --no-headers | awk '{print $3}')"
echo "Load Balancer IP $IP"

然而，这个有它的问题，我需要在运行这个命令之前添加睡眠，以确保已经分配了IP。我不能确定增加的睡眠时间是否足够。 实际上需要像这样的smth，但对于我的入口负载均衡器IP

output "google_container_cluster_endpoint" {
  value = "${google_container_cluster.k8s.endpoint}"
}

output "google_container_cluster_master_version" {
  value = "${google_container_cluster.k8s.master_version}"
}

---

经过漫长的周六，我终于找到了解决问题的办法。我遇到了几乎相同的问题，所以这里是我的解决方案，当然可以改进

我分为两部分：

1.-我将使用local exec运行脚本，解决在负载平衡器中等待有效IP的问题 2.-Terraform，使用外部数据源调用一个以json格式应答的“程序”。我的“程序”是一个获取IP的bash脚本。因此，我在一个变量中有了所需的数据

我这样做是因为我不知道如何使用外部数据源调试问题，而且我遇到了“奇怪的事情”

首先，我运行代码以等待有效的IP。Terraform调用本地exec

---

---

这就是我使用的脚本

---

---

在我知道我已经准备好IP之后。我只需要抓住它。注意所依赖的，它控制着一旦我的资源（google\u container\u cluster.tests）被创建，我就会抓取数据，而不是在他想要的时候。测试一下。这很棘手

---

---

这是test jq.sh（test cause是我第一次使用：S），我调用该脚本以json格式打印数据

---

---

希望能有帮助。至少我已经解决了我的问题。

---

这可以通过以下方式仅使用kubectl来完成：

#!/bin/bash

while true;
do
  kubectl get svc -n istio-system istio-ingressgateway -o jsonpath='{"{\"ip\": "}{"\""}{.status.loadBalancer.ingress[0].ip}{"\"}"}' | grep -E -o "([0-9]{1,3}[\.]){3}[0-9]{1,3}" >/dev/null

  if [[ $? -eq 0 ]]; then
      kubectl get svc -n istio-system istio-ingressgateway -o jsonpath='{"{\"ip\": "}{"\""}{.status.loadBalancer.ingress[0].ip}{"\"}"}'
      exit 0
  fi
done

现在，您可以按如下方式为外部数据资源编写脚本：

data "external" "external-public-ip" {
  program = ["sh", "get-ip.sh" ]
  depends_on = [kubernetes_service.foo]
}

output "external-public-ip" {
  value = "${data.external.external-public-ip.result}"
}

---

我已经设法以完全声明的方式获得外部入口ip。它基于不同的供应商，包括azurerm、kubernetes和helm。我的目标是Azure Kubernetes服务，但解决方案是云不可知的

解决方案说明：

创建入口后，使用kubernetes提供程序连接群集。Kubernetes允许读取外部ip之类的服务数据

供应商概述：

• azurerm提供程序用于Azure通信
  • 可以通过不同的提供者创建Kubernetes（K8s）
• 头盔供应器用于入口安装，
  • 可以使用不同的方法创建入口
• kubernetes提供者允许我查询负载平衡器服务

短片段

provider "kubernetes" { }

provider "helm" { }

resource "helm_release" "nginx-ingress" {
  name             = "nginx-ingress"
  namespace        = "nginx-ingress"
  create_namespace = true
  repository       = "https://kubernetes-charts.storage.googleapis.com"
  chart            = "nginx-ingress"

  set {
    name  = "controller.replicaCount"
    value = "2"
  }
}

data "kubernetes_service" "service_ingress" {
  metadata {
    name      = "nginx-ingress-controller"
    namespace = "nginx-ingress"
  }

  depends_on = [ helm_release.nginx-ingress ] 
}

output "ip" {
  value = data.kubernetes_service.service_ingress.load_balancer_ingress.0.ip
}

variable "subscription_id" {
  type = string
}

variable "client_id" {
  type = string
}

variable "client_secret" {
  type = string
}

variable "tenant_id" {
  type = string
}

variable "resource_location"{
  type = string
}

terraform {
  required_providers {
    azurerm = {
      source = "hashicorp/azurerm"
      version = "2.29.0"
    }

    kubernetes = {
      source = "hashicorp/kubernetes"
      version = "1.13.2"
    }

    helm = {
      source = "hashicorp/helm"
      version = "1.3.1"
    }
  }
}

provider "azurerm" {
  subscription_id = var.subscription_id
  client_id       = var.client_id
  client_secret   = var.client_secret
  tenant_id       = var.tenant_id
  features {}
}

provider "kubernetes" {
  load_config_file       = "false"
  host                   = azurerm_kubernetes_cluster.aks.kube_config.0.host
  client_certificate     = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.client_certificate)
  client_key             = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.client_key)
  cluster_ca_certificate = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.cluster_ca_certificate)
}

provider "helm" {
  kubernetes {
    load_config_file       = "false"
    host                   = azurerm_kubernetes_cluster.aks.kube_config.0.host
    client_certificate     = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.client_certificate)
    client_key             = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.client_key)
    cluster_ca_certificate = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.cluster_ca_certificate)
  }
}

data "kubernetes_service" "service_ingress" {
  metadata {
    name      = "nginx-ingress-controller"
    namespace = "nginx-ingress"
  }

  depends_on = [ helm_release.nginx-ingress ] 
}

resource "azurerm_resource_group" "rg" {
  name     = "myapp"
  location = var.resource_location
}

resource "azurerm_kubernetes_cluster" "aks" {
  name                = "myapp"
  location            = var.resource_location
  resource_group_name = azurerm_resource_group.rg.name
  dns_prefix          = "myapp"
  kubernetes_version  = "1.17.11"

  default_node_pool {
    name            = "default"
    node_count      = 2
    vm_size         = "Standard_B2s"
    os_disk_size_gb = 30
    type = "VirtualMachineScaleSets"
    enable_auto_scaling = false
  }

  service_principal {
    client_id     = var.client_id
    client_secret = var.client_secret
  }

  role_based_access_control {
    enabled = true
  }
}

resource "helm_release" "nginx-ingress" {
  name             = "nginx-ingress"
  namespace        = "nginx-ingress"
  create_namespace = true
  repository       = "https://kubernetes-charts.storage.googleapis.com"
  chart            = "nginx-ingress"

  set {
    name  = "controller.replicaCount"
    value = "2"
  }

  set {
    name  = "controller.nodeSelector.kubernetes\\.io/os"
    value = "linux"
  }

  set {
    name  = "defaultBackend.nodeSelector.kubernetes\\.io/os"
    value = "linux"
  }
}

output "ip" {
  value = data.kubernetes_service.service_ingress.load_balancer_ingress.0.ip
}

完成代码片段

provider "kubernetes" { }

provider "helm" { }

resource "helm_release" "nginx-ingress" {
  name             = "nginx-ingress"
  namespace        = "nginx-ingress"
  create_namespace = true
  repository       = "https://kubernetes-charts.storage.googleapis.com"
  chart            = "nginx-ingress"

  set {
    name  = "controller.replicaCount"
    value = "2"
  }
}

data "kubernetes_service" "service_ingress" {
  metadata {
    name      = "nginx-ingress-controller"
    namespace = "nginx-ingress"
  }

  depends_on = [ helm_release.nginx-ingress ] 
}

output "ip" {
  value = data.kubernetes_service.service_ingress.load_balancer_ingress.0.ip
}

variable "subscription_id" {
  type = string
}

variable "client_id" {
  type = string
}

variable "client_secret" {
  type = string
}

variable "tenant_id" {
  type = string
}

variable "resource_location"{
  type = string
}

terraform {
  required_providers {
    azurerm = {
      source = "hashicorp/azurerm"
      version = "2.29.0"
    }

    kubernetes = {
      source = "hashicorp/kubernetes"
      version = "1.13.2"
    }

    helm = {
      source = "hashicorp/helm"
      version = "1.3.1"
    }
  }
}

provider "azurerm" {
  subscription_id = var.subscription_id
  client_id       = var.client_id
  client_secret   = var.client_secret
  tenant_id       = var.tenant_id
  features {}
}

provider "kubernetes" {
  load_config_file       = "false"
  host                   = azurerm_kubernetes_cluster.aks.kube_config.0.host
  client_certificate     = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.client_certificate)
  client_key             = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.client_key)
  cluster_ca_certificate = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.cluster_ca_certificate)
}

provider "helm" {
  kubernetes {
    load_config_file       = "false"
    host                   = azurerm_kubernetes_cluster.aks.kube_config.0.host
    client_certificate     = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.client_certificate)
    client_key             = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.client_key)
    cluster_ca_certificate = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.cluster_ca_certificate)
  }
}

data "kubernetes_service" "service_ingress" {
  metadata {
    name      = "nginx-ingress-controller"
    namespace = "nginx-ingress"
  }

  depends_on = [ helm_release.nginx-ingress ] 
}

resource "azurerm_resource_group" "rg" {
  name     = "myapp"
  location = var.resource_location
}

resource "azurerm_kubernetes_cluster" "aks" {
  name                = "myapp"
  location            = var.resource_location
  resource_group_name = azurerm_resource_group.rg.name
  dns_prefix          = "myapp"
  kubernetes_version  = "1.17.11"

  default_node_pool {
    name            = "default"
    node_count      = 2
    vm_size         = "Standard_B2s"
    os_disk_size_gb = 30
    type = "VirtualMachineScaleSets"
    enable_auto_scaling = false
  }

  service_principal {
    client_id     = var.client_id
    client_secret = var.client_secret
  }

  role_based_access_control {
    enabled = true
  }
}

resource "helm_release" "nginx-ingress" {
  name             = "nginx-ingress"
  namespace        = "nginx-ingress"
  create_namespace = true
  repository       = "https://kubernetes-charts.storage.googleapis.com"
  chart            = "nginx-ingress"

  set {
    name  = "controller.replicaCount"
    value = "2"
  }

  set {
    name  = "controller.nodeSelector.kubernetes\\.io/os"
    value = "linux"
  }

  set {
    name  = "defaultBackend.nodeSelector.kubernetes\\.io/os"
    value = "linux"
  }
}

output "ip" {
  value = data.kubernetes_service.service_ingress.load_balancer_ingress.0.ip
}

---

您正在使用哪些地形模块？你是如何配置集群本身的？我不使用任何模块，使用资源“google_container_cluster”创建kubernetes集群有一个和一个匹配的，但我不认为有一个好方法可以强制Terraform等待负载平衡器的存在。类似于

output“IP”{value=${kubernetes_service.load_balancer_service.load_balancer_ingres.0.ip}

不起作用？很好。这并不能解决我的问题，因为我想在输出中得到这个结果的主要原因是不必运行shell脚本。我当前的配置也使用shell脚本来获得负载平衡器IP。但是，如果你想将shell脚本的输出作为变量，不需要使用外部数据，你可以使用TF_VARNAME来从shell中定义terraform变量。另一种获取负载平衡器IP作为输出变量的方法是使用此terraform模块（）将ingress部署为一个单独的terraform资源，并将负载平衡器IP地址作为其输出。但是，如果没有限制使用shell脚本，您的解决方法看起来也不错。太好了！我只需将

IP

输出修改为

data.kubernetes\u service.service\u ingress.status[0]。负载平衡器[0]。ingress[0].ip

我也这么做了，但是设置依赖项不足以强制等待它的创建，tf只是将其作为

null

获取，因此第一个

apply

失败，第二个成功读取数据资源并输出值。恼人的是，在使用wait资源进行一些测试后，该值似乎没有重新生成如果它从一开始就不存在的话，那它根本就不存在。可能有一些变化，但是解决方案是完全可操作的。依赖性工作正常。有人有同样的问题吗？

provider "kubernetes" { }

provider "helm" { }

resource "helm_release" "nginx-ingress" {
  name             = "nginx-ingress"
  namespace        = "nginx-ingress"
  create_namespace = true
  repository       = "https://kubernetes-charts.storage.googleapis.com"
  chart            = "nginx-ingress"

  set {
    name  = "controller.replicaCount"
    value = "2"
  }
}

data "kubernetes_service" "service_ingress" {
  metadata {
    name      = "nginx-ingress-controller"
    namespace = "nginx-ingress"
  }

  depends_on = [ helm_release.nginx-ingress ] 
}

output "ip" {
  value = data.kubernetes_service.service_ingress.load_balancer_ingress.0.ip
}

variable "subscription_id" {
  type = string
}

variable "client_id" {
  type = string
}

variable "client_secret" {
  type = string
}

variable "tenant_id" {
  type = string
}

variable "resource_location"{
  type = string
}

terraform {
  required_providers {
    azurerm = {
      source = "hashicorp/azurerm"
      version = "2.29.0"
    }

    kubernetes = {
      source = "hashicorp/kubernetes"
      version = "1.13.2"
    }

    helm = {
      source = "hashicorp/helm"
      version = "1.3.1"
    }
  }
}

provider "azurerm" {
  subscription_id = var.subscription_id
  client_id       = var.client_id
  client_secret   = var.client_secret
  tenant_id       = var.tenant_id
  features {}
}

provider "kubernetes" {
  load_config_file       = "false"
  host                   = azurerm_kubernetes_cluster.aks.kube_config.0.host
  client_certificate     = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.client_certificate)
  client_key             = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.client_key)
  cluster_ca_certificate = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.cluster_ca_certificate)
}

provider "helm" {
  kubernetes {
    load_config_file       = "false"
    host                   = azurerm_kubernetes_cluster.aks.kube_config.0.host
    client_certificate     = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.client_certificate)
    client_key             = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.client_key)
    cluster_ca_certificate = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.cluster_ca_certificate)
  }
}

data "kubernetes_service" "service_ingress" {
  metadata {
    name      = "nginx-ingress-controller"
    namespace = "nginx-ingress"
  }

  depends_on = [ helm_release.nginx-ingress ] 
}

resource "azurerm_resource_group" "rg" {
  name     = "myapp"
  location = var.resource_location
}

resource "azurerm_kubernetes_cluster" "aks" {
  name                = "myapp"
  location            = var.resource_location
  resource_group_name = azurerm_resource_group.rg.name
  dns_prefix          = "myapp"
  kubernetes_version  = "1.17.11"

  default_node_pool {
    name            = "default"
    node_count      = 2
    vm_size         = "Standard_B2s"
    os_disk_size_gb = 30
    type = "VirtualMachineScaleSets"
    enable_auto_scaling = false
  }

  service_principal {
    client_id     = var.client_id
    client_secret = var.client_secret
  }

  role_based_access_control {
    enabled = true
  }
}

resource "helm_release" "nginx-ingress" {
  name             = "nginx-ingress"
  namespace        = "nginx-ingress"
  create_namespace = true
  repository       = "https://kubernetes-charts.storage.googleapis.com"
  chart            = "nginx-ingress"

  set {
    name  = "controller.replicaCount"
    value = "2"
  }

  set {
    name  = "controller.nodeSelector.kubernetes\\.io/os"
    value = "linux"
  }

  set {
    name  = "defaultBackend.nodeSelector.kubernetes\\.io/os"
    value = "linux"
  }
}

output "ip" {
  value = data.kubernetes_service.service_ingress.load_balancer_ingress.0.ip
}