Merge pull request 'remove cat' (#244) from fenix-admin into main

Reviewed-on: fenix-gitea-admin/iac-opentofu-private#244

.gitea/workflows/ci-test.yaml

@@ -15,8 +15,21 @@ jobs:
       PM_API_TOKEN_SECRET: ${{ secrets.PM_API_TOKEN_SECRET }}
       BW_EMAIL: ${{ secrets.BW_EMAIL }}
       BW_PASSWORD: ${{ secrets.BW_PASSWORD }}
+      BW_CLIENTID: ${{ secrets.BW_CLIENTID }}
+      BW_CLIENTSECRET: ${{ secrets.BW_CLIENTSECRET }}
       
     steps:
+
+      - name: Verificar se commit pede por [deploy-opentofu]
+        if: "!contains(gitea.event.head_commit.message, '[deploy-opentofu]')"
+        run: |
+          echo "Commit não contém [deploy-opentofu], a pipeline será ignorada."
+          exit 1
+
+      - name: Executar deploy opentofu
+        run: echo "Commit contém [deploy-opentofu], a pipeline será executada."
+
+
       - name: Updating apt-get
         run: |
           apt-get update -y
@@ -26,12 +39,12 @@ jobs:
         with:
           path: infra/iac
 
-      - name: cloning iac secrets repository
-        uses: actions/checkout@v4
-        with:
-          repository: fenix-gitea-admin/iac-teste-secrets
-          token: ${{ secrets.GGITEA_TOKEN }}
-          path: infra/secrets
+      #- name: cloning iac secrets repository
+      #  uses: actions/checkout@v4
+      #  with:
+      #    repository: fenix-gitea-admin/iac-opentofu-private-secrets
+      #    token: ${{ secrets.GGITEA_TOKEN }}
+      #    path: infra/secrets
 
 
       - name: Install cloudflare prerequisites
@@ -109,22 +122,63 @@ jobs:
           sleep 3
           cat dante.log
           
-      - name: vaultwarden login
+      #- name: vaultwarden login
+      #  working-directory: infra/iac
+      #  run: |
+      #    bw config server https://vaultwarden.fenix-dev.com
+      #    #BW_SESSION=$(bw login)
+      #    bw login --apikey
+      #    BW_SESSION=$(bw unlock "$BW_PASSWORD" --raw)
+          
+
+      - name: vaultwarden getsecrets
         working-directory: infra/iac
         run: |
-          BW_SESSION=$(bw login)
+          bw config server https://vaultwarden.fenix-dev.com
+          bw login --apikey
+          BW_SESSION=$(bw unlock "$BW_PASSWORD" --raw)
+          
+          # Ler o arquivo de referência
+          for secret in $(jq -c '.secrets[]' secrets/vault-secrets-map.json); do
+            name=$(echo "$secret" | jq -r '.name')
+            type=$(echo "$secret" | jq -r '.type')
+            output=$(echo "$secret" | jq -r '.output')
+            echo "$name $type $output"
+            item_id=$(bw get item "$name"  | jq -r '.id')
+            echo "$item_id"
+            mkdir -p "$(dirname "$output")"
+            echo "dir made"
+            if [ "$type" == "attachment" ]; then
+              echo "attachment get"
+              bw get attachment "$output" --itemid "$item_id" --output "$output"  --session "$BW_SESSION" 
+            elif [ "$type" == "note" ]; then
+              echo "note get"
+              bw get item "$name"  --session "$BW_SESSION"  | jq -r '.notes' > "$output"
+            fi
+          done
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: |
+          pip install ruamel.yaml
 
       - name: Init OpenTofu
         working-directory: infra/iac
         run: |
-          tofu init
+          tofu init     
 
       - name: Plan
         working-directory: infra/iac
         run: |
-          # Find all .auto.tfvars in the secrets folder
+          # Find all .tfvars in the secrets folder
           VAR_FILES=""
-          for f in ../secrets/secrets/*.auto.tfvars; do
+          ls -la ../secrets/secrets
+          for f in ../secrets/secrets/*.tfvars; do
+            echo "found - $f"
             VAR_FILES="$VAR_FILES -var-file=$f"
           done
           tofu plan $VAR_FILES -out=./tfplan  | tee ./tfplan.txt

cloud-init-base.yaml

@@ -0,0 +1,19 @@
+locale: "pt_PT.UTF-8"
+keyboard:
+    layout: "pt"
+    variant: "nodeadkeys"
+
+users:
+    - default
+
+ssh_pwauth: true
+
+package_update: true
+packages:
+    - qemu-guest-agent
+    - net-tools
+    - curl
+runcmd:
+    - systemctl enable qemu-guest-agent
+    - systemctl start qemu-guest-agent
+    - echo "done" > /tmp/cloud-config.done

cloud-init-vm.yaml.tftpl

@@ -0,0 +1,36 @@
+package_update: true
+%{ if length(each.value.extra_packages) > 0 ~}
+packages:
+%{ for pkg in each.value.extra_packages ~}
+  - ${pkg}
+%{ endfor ~}
+%{ endif ~}
+
+%{ if length(each.value.extra_users) > 0 ~}
+users:
+%{ for u in each.value.extra_users ~}
+  - name: ${u.name}
+    groups: 
+%{ if length(u.groups) > 0 ~}
+%{ for g in u.groups ~}
+      - ${g}
+%{ endfor ~}
+%{ endif ~}
+    shell: /bin/bash
+    sudo: ALL=(ALL) NOPASSWD:ALL
+%{ endfor ~}
+
+chpasswd:
+  list: ${"|"}
+%{ for u in each.value.extra_users ~}
+    ${u.name}:${u.password}
+%{ endfor ~}
+  expire: false
+%{ endif ~}
+
+%{ if length(each.value.extra_runcmd) > 0 ~}
+runcmd:
+%{ for cmd in each.value.extra_runcmd ~}
+  - ${cmd}
+%{ endfor ~}
+%{ endif ~}

cloud-init-wrapper.yaml.tftpl

@@ -0,0 +1,2 @@
+#cloud-config
+${content}

documentation/Dockerfile

@@ -0,0 +1,23 @@
+FROM ghcr.io/opentofu/opentofu:1.9-minimal AS tofu
+
+FROM ubuntu:24.04
+ 
+# Copy the tofu binary
+COPY --from=tofu /usr/local/bin/tofu /usr/local/bin/tofu
+
+# Install dependencies
+RUN apt-get update && apt-get install -y \
+    git \
+    curl \
+    nodejs \
+    npm \
+    unzip \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN curl -L -o /tmp/bw.zip https://github.com/bitwarden/cli/releases/download/v1.22.1/bw-linux-1.22.1.zip \
+    && unzip /tmp/bw.zip -d /usr/local/bin \
+    && chmod +x /usr/local/bin/bw \
+    && rm /tmp/bw.zip
+
+
+WORKDIR /workspace

documentation/start.txt

@@ -1,9 +1,9 @@
-https://spacelift.io/blog/opentofu-tutorial - explaining language of opentofu
+https://spacelift.io/blog/opentofu-tutorial - explaining language of opentofu 
 
 https://opentofu.org/docs/intro/ - quick start and explaning who to work in team
 
 https://opentofu.org/docs/intro/ - CICD for opentofu explained
-
+ 
 
 
 tofu init

main.tf

@@ -1,9 +1,5 @@
 terraform {
   required_providers {
-    random = {
-      source  = "hashicorp/random"
-      version = "~> 3.6"
-    }
     vaultwarden = {
       source = "ottramst/vaultwarden"
       version = "0.4.4"
@@ -24,5 +20,3 @@ terraform {
     lock    = true                                        # enable state locking
   }
 }
-
-provider "random" {}

merge_yaml.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+from ruamel.yaml import YAML
+import sys
+import json
+from collections.abc import Mapping
+
+def deep_merge_yaml(dict1, dict2):
+    result = dict1.copy()
+    for key, value in dict2.items():
+        if key in result:
+            if isinstance(result[key], list) and isinstance(value, list):
+                result[key] = result[key] + value
+            elif isinstance(result[key], Mapping) and isinstance(value, Mapping):
+                result[key] = deep_merge_yaml(result[key], value)
+            else:
+                result[key] = value
+        else:
+            result[key] = value
+    return result
+
+def main():
+    input_data = json.load(sys.stdin)
+    file1 = input_data["file1"]
+    file2 = input_data["file2"]
+
+    yaml = YAML()
+    yaml.indent(mapping=2, sequence=4, offset=2)
+
+    with open(file1, "r") as f1, open(file2, "r") as f2:
+        yaml1 = yaml.load(f1)
+        yaml2 = yaml.load(f2)
+
+    merged = deep_merge_yaml(yaml1, yaml2)
+    
+    from io import StringIO
+    output = StringIO()
+    yaml.dump(merged, output)
+
+    print(json.dumps({
+        "merged_yaml": output.getvalue()
+    }))
+
+
+if __name__ == "__main__":
+    main()

proxmox.tf

@@ -30,32 +30,58 @@ resource "proxmox_virtual_environment_file" "cloud_init_yaml" {
 
   source_raw {
     file_name = "user-data-cloud-config.yaml"
-    data = <<-EOF
-    #cloud-config
-    users:
-      - default
-      - name: testeuser
-        groups: sudo
-        shell: /bin/bash
-        sudo: ALL=(ALL) NOPASSWD:ALL
+    data = file("${path.module}/cloud-init-base.yaml")
+  }
+}
 
-    ssh_pwauth: true
 
-    chpasswd:
-      list: |
-        testeuser:testepassword
-      expire: false
+locals {
+  rendered_yaml_per_vm ={
+    for vm in var.proxmox_k8s_vms : vm.name =>
+      templatefile("${path.module}/cloud-init-vm.yaml.tftpl", 
+    {
+      each = {
+        value = vm
+      }
+    })
+  } 
+}
 
-    package_update: true
-    packages:
-      - qemu-guest-agent
-      - net-tools
-      - curl
-    runcmd:
-      - systemctl enable qemu-guest-agent
-      - systemctl start qemu-guest-agent
-      - echo "done" > /tmp/cloud-config.done
-    EOF
+
+data "external" "merged_yaml" {
+  for_each = { for vm in var.proxmox_k8s_vms : vm.name => vm }
+
+  depends_on = [local_file.rendered_yaml_file]
+
+  program = ["python3", "${path.module}/merge_yaml.py"]
+
+  query = {
+    file1 = "${path.module}/cloud-init-base.yaml"
+    file2 = "${path.module}/cloud-init-extra-${each.key}.yaml"
+  }
+}
+
+
+resource "local_file" "rendered_yaml_file" {
+  for_each = { for vm in var.proxmox_k8s_vms : vm.name => vm }
+
+  content  = local.rendered_yaml_per_vm[each.key]
+  filename = "${path.module}/cloud-init-extra-${each.key}.yaml"
+}
+
+
+# Gerar um snippet cloud-init por VM
+resource "proxmox_virtual_environment_file" "vm_user_data" {
+  for_each = { for vm in var.proxmox_k8s_vms : vm.name => vm }
+
+  node_name    = "fenix"
+  datastore_id = "local-snippets"
+  content_type = "snippets"
+  source_raw {
+    file_name = "cloud-init-iac-k8s-${each.key}.yaml"
+    data      = templatefile("${path.module}/cloud-init-wrapper.yaml.tftpl", {
+      content = data.external.merged_yaml[each.key].result["merged_yaml"]
+    })
   }
 }
 
@@ -85,13 +111,6 @@ resource "proxmox_virtual_environment_vm" "proxmox-kubernetes-VM-template" {
     dedicated = 4096
   }
 
-  # Configuração do disco rígido
-  disk {
-    datastore_id = "local-lvm"
-    interface    = "scsi1"
-    size         = 64
-  }
-
   disk {
     datastore_id = "local-lvm"
     file_id      = proxmox_virtual_environment_download_file.latest_ubunto_cloud_img.id
@@ -102,32 +121,59 @@ resource "proxmox_virtual_environment_vm" "proxmox-kubernetes-VM-template" {
   # Configuração da interface de rede
   network_device {
     bridge = "vmbr0"
-    model  = "virtio"
   }
 
 initialization {
-  ip_config {
-    ipv4 {
-    address = "dhcp"  # IP estático + máscara de rede
+    dns {
+      servers = ["1.1.1.1"]
+    }
+    ip_config {
+      ipv4 {
+        address = "dhcp"
+      }
     }
-  }
   user_data_file_id = proxmox_virtual_environment_file.cloud_init_yaml.id
  }
 }
 
 
-resource "proxmox_virtual_environment_vm" "VM-Kubernetes-01" {
+resource "proxmox_virtual_environment_vm" "k8s_vms" {
+  for_each  = { for vm in var.proxmox_k8s_vms : vm.name => vm }
   depends_on = [proxmox_virtual_environment_vm.proxmox-kubernetes-VM-template]
-  name      = "VM-Kubernetes-01"
-  node_name = "fenix"
-
+  name      = each.value.name
+  node_name = each.value.node_name
+  vm_id = each.value.vm_id
+  
   clone {
     vm_id = proxmox_virtual_environment_vm.proxmox-kubernetes-VM-template.id
   }
-  agent {
-    enabled = true
+
+  cpu {
+    cores = each.value.cores
   }
-}
 
+  memory {
+    dedicated = each.value.memory
+  }
 
+  disk {
+    datastore_id = each.value.data_store
+    size         = each.value.disk_size
+    interface    = "scsi1"
+  }
 
+  initialization {
+      ip_config {
+        ipv4 {
+          address = each.value.ip
+          gateway = each.value.gateway
+        }
+      }
+
+      user_data_file_id = proxmox_virtual_environment_file.vm_user_data[each.key].id
+    }
+
+    agent {
+      enabled = true
+    }
+  }

proxmox.variables.tf

@@ -32,3 +32,25 @@ variable "proxmox_apikey" {
 variable "node_name" {
   default = "fenix"
 }
+
+# Lista de VMs (override de valores específicos)
+variable "proxmox_k8s_vms" {
+  type = list(object({
+    name      = string
+    vm_id     = number
+    node_name  = string
+    ip        = string
+    cores     = optional(number)
+    memory    = optional(number)
+    data_store = optional(string)
+    gateway = string
+    disk_size = optional(number)
+    extra_users = optional(list(object({
+      name     = string
+      password = string
+      groups   = list(string)
+    })))
+    extra_packages = optional(list(string))
+    extra_runcmd   = optional(list(string))
+  }))
+}

secrets/vault-secrets-map.json

@@ -0,0 +1,19 @@
+{
+  "secrets": [
+    { 
+      "name": "iac.opentofu.consul.secrets",
+      "type": "note",
+      "output": "../secrets/secrets/consul.secrets.tfvars"
+    },
+    {
+      "name": "iac.opentofu.proxmox.secrets",
+      "type": "note",
+      "output": "../secrets/secrets/proxmox.secrets.tfvars"
+    },
+    {
+      "name": "iac.opentofu.vaultwarden.secrets",
+      "type": "note",
+      "output": "../secrets/secrets/vaultwarden.secrets.tfvars"
+    }
+  ]
+}

secrets/vaultwarden.secrets.tfvars

@@ -3,4 +3,4 @@ vaultwarden_email    = "admin@example.com"
 vaultwarden_master_password = "SuperSecretMasterPassword"
 vaultwarden_admin_token = "tokenadmin"
 vaultwarden_client_id = "clientid"
-vaultwarden_client_secret = "clientsecret"
+vaultwarden_client_secret = "clientsecret"