My homelab from scratch (Part 9): OpenTofu, CI/CD, or when the pipeline reveals your hidden dependencies

ansible/roles/pve_api_token/
├── defaults/main.yml   # user, roles, token id
└── tasks/main.yml      # user + ACL + token creation, all idempotent

# ansible/roles/pve_api_token/defaults/main.yml
---
pve_api_token_user: tofu@pve
pve_api_token_id: opentofu
pve_api_token_roles:
  - PVEAdmin
  - PVESysAdmin
pve_api_token_path: /

# ansible/roles/pve_api_token/tasks/main.yml
---
- name: Check whether API user exists
  ansible.builtin.command: "pveum user list --output-format json"
  register: pve_api_token_users
  changed_when: false

- name: Create API user
  ansible.builtin.command: "pveum user add {{ pve_api_token_user }} --comment 'OpenTofu automation'"
  when: pve_api_token_user not in (pve_api_token_users.stdout | from_json | map(attribute='userid') | list)
  changed_when: true

- name: Read current ACL
  ansible.builtin.command: "pveum acl list --output-format json"
  register: pve_api_token_acl
  changed_when: false

- name: Grant roles to API user
  ansible.builtin.command: >-
    pveum acl modify {{ pve_api_token_path }}
    --users {{ pve_api_token_user }}
    --roles {{ item }}
  loop: "{{ pve_api_token_roles }}"
  when: >-
    pve_api_token_acl.stdout | from_json
    | selectattr('ugid', 'equalto', pve_api_token_user)
    | selectattr('roleid', 'equalto', item)
    | list | length == 0
  changed_when: true

- name: Check whether token exists
  ansible.builtin.command: "pveum user token list {{ pve_api_token_user }} --output-format json"
  register: pve_api_token_existing
  changed_when: false

- name: Create API token
  ansible.builtin.command: >-
    pveum user token add {{ pve_api_token_user }} {{ pve_api_token_id }}
    --privsep 0
    --output-format json
  register: pve_api_token_new
  when: pve_api_token_id not in (pve_api_token_existing.stdout | from_json | map(attribute='tokenid') | list)
  changed_when: true

- name: Show token secret
  # noqa: no-handler - one-shot display of a secret shown only at creation
  ansible.builtin.debug:
    msg: >-
      Token created. Store it now, it will not be shown again:
      {{ (pve_api_token_new.stdout | from_json).value }}
  when: pve_api_token_new.changed

proxmox_api_token_id       # tofu@pve!opentofu
proxmox_api_token_secret   # the generated UUID
gitlab_project_id          # the GitLab project id
gitlab_token               # project PAT, api scope
tofu_state_passphrase      # openssl rand -base64 32
gitlab_runner_token        # glrt-...

TOKEN_ID=$(sops -d secrets/tofu.sops.yaml | yq -r '.proxmox_api_token_id')
TOKEN_SECRET=$(sops -d secrets/tofu.sops.yaml | yq -r '.proxmox_api_token_secret')
AUTH="PVEAPIToken=${TOKEN_ID}=${TOKEN_SECRET}"
curl -sk -H "Authorization: ${AUTH}" https://192.168.3.11:8006/api2/json/nodes | jq -r '.data[].node'

pve01
pve02
pve03

# tofu/env.sh
#!/usr/bin/env bash

_tofu_env() {
  local stack="${1:-core}"
  local root secrets plain project_id gl_token address passphrase

  root="$(git rev-parse --show-toplevel 2>/dev/null)"
  if [ -z "$root" ]; then
    echo "error: run from inside the git repository" >&2
    return 1
  fi

  secrets="$root/secrets/tofu.sops.yaml"
  plain="$(sops -d "$secrets")" || { echo "error: cannot decrypt $secrets" >&2; return 1; }
  _get() { printf '%s' "$plain" | yq -r ".$1"; }

  project_id="$(_get gitlab_project_id)"
  gl_token="$(_get gitlab_token)"
  passphrase="$(_get tofu_state_passphrase)"
  address="https://gitlab.com/api/v4/projects/${project_id}/terraform/state/${stack}"

  export TF_HTTP_ADDRESS="$address"
  export TF_HTTP_LOCK_ADDRESS="${address}/lock"
  export TF_HTTP_UNLOCK_ADDRESS="${address}/lock"
  export TF_HTTP_LOCK_METHOD="POST"
  export TF_HTTP_UNLOCK_METHOD="DELETE"
  export TF_HTTP_USERNAME="gitlab-ci-token"
  export TF_HTTP_PASSWORD="$gl_token"

  export TF_ENCRYPTION="key_provider \"pbkdf2\" \"state\" {
  passphrase = \"${passphrase}\"
}
method \"aes_gcm\" \"state\" {
  keys = key_provider.pbkdf2.state
}
state {
  method = method.aes_gcm.state
  enforced = true
}
plan {
  method = method.aes_gcm.state
  enforced = true
}"

  export PROXMOX_VE_API_TOKEN="$(_get proxmox_api_token_id)=$(_get proxmox_api_token_secret)"

  local bootstrap
  bootstrap="$(sops -d "$root/secrets/bootstrap.sops.yaml")" || return 1
  export PROXMOX_VE_SSH_USERNAME="root"
  export PROXMOX_VE_SSH_PRIVATE_KEY="$(printf '%s' "$bootstrap" | yq -r '.ssh_private_key')"
  export TF_VAR_ssh_public_key="$(printf '%s' "$bootstrap" | yq -r '.ssh_public_key')"

  # ... (more TF_VARs will come for the next phases)
  echo "OpenTofu environment ready for stack: $stack"
}

_tofu_env "$@"
unset -f _tofu_env

tofu/stacks/core/
├── versions.tf    # required_version, bpg provider, empty http backend
├── providers.tf   # endpoint, ssh
├── variables.tf   # the stack's variables
├── main.tf        # verification data source
└── runner.tf      # (step 4)

# tofu/stacks/core/versions.tf
terraform {
  required_version = ">= 1.12.0"

  required_providers {
    proxmox = {
      source  = "bpg/proxmox"
      version = "~> 0.100"
    }
    # (the Talos, Helm and Kubernetes providers will come in the next phases)
  }

  # Credentials and encryption come from env vars (see tofu/env.sh)
  backend "http" {}
}

# tofu/stacks/core/providers.tf
provider "proxmox" {
  endpoint = var.proxmox_endpoint
  insecure = true

  # API token and SSH credentials come from environment (see tofu/env.sh)
  ssh {
    agent = false
  }
}

# tofu/stacks/core/variables.tf
variable "proxmox_endpoint" {
  description = "Proxmox VE API endpoint"
  type        = string
  default     = "https://192.168.3.11:8006/"
}

variable "runner_node" {
  description = "Node hosting the GitLab runner container"
  type        = string
  default     = "pve01"
}

variable "runner_template" {
  description = "LXC template file id (must exist on runner_node)"
  type        = string
  default     = "local:vztmpl/debian-13-standard_13.6-1_amd64.tar.zst"
}

variable "runner_vm_id" {
  description = "Container id"
  type        = number
  default     = 200
}

variable "runner_ip" {
  description = "Static IPv4 with CIDR"
  type        = string
  default     = "192.168.3.50/24"
}

variable "runner_gateway" {
  type    = string
  default = "192.168.3.1"
}

variable "runner_dns" {
  type    = string
  default = "1.1.1.1"
}

variable "storage_pool" {
  description = "ZFS pool backing container volumes"
  type        = string
  default     = "nvme-vm"
}

variable "ssh_public_key" {
  description = "Public key injected into containers (set via TF_VAR_ssh_public_key)"
  type        = string
}

# tofu/stacks/core/main.tf
data "proxmox_virtual_environment_nodes" "cluster" {}

output "cluster_nodes" {
  description = "Nodes seen through the API"
  value       = data.proxmox_virtual_environment_nodes.cluster.names
}

curl -s -u "gitlab-ci-token:$TF_HTTP_PASSWORD" "$TF_HTTP_ADDRESS" | head -c 400

{"serial":1,"lineage":"31f7af38-...","meta":{"key_provider.pbkdf2.state":
"eyJzYWx0IjoiRDlxTUpDUnZ4WURkUHBIZWF3ZURmWlhOblZLK3ZoNExDM2Z2Z3U0NittOD0i
LCJpdGVyYXRpb25zIjo2MDAwMDAsImhhc2hfZnVuY3Rpb24iOiJzaGE1MTIiLCJrZXlfbGVuZ
3RoIjozMn0="},"encrypted_data":"TfPgZfdLvDWiHTp9WWGotfF730BOX2A+..."}

ansible/roles/pve_lxc_template/
├── defaults/main.yml   # template pattern + target storage
└── tasks/main.yml      # catalogue refresh + idempotent download

# ansible/roles/pve_lxc_template/defaults/main.yml
---
pve_lxc_template_pattern: debian-13-standard
pve_lxc_template_storage: local

# ansible/roles/pve_lxc_template/tasks/main.yml
---
- name: Refresh appliance catalog
  ansible.builtin.command: pveam update
  changed_when: false

- name: Find latest matching template
  ansible.builtin.shell: |
    set -o pipefail
    pveam available --section system \
      | awk '{print $2}' \
      | grep "^{{ pve_lxc_template_pattern }}" \
      | sort -V | tail -1
  args:
    executable: /bin/bash
  register: pve_lxc_template_latest
  changed_when: false
  failed_when: pve_lxc_template_latest.stdout | length == 0

- name: List downloaded templates
  ansible.builtin.command: "pveam list {{ pve_lxc_template_storage }}"
  register: pve_lxc_template_present
  changed_when: false

- name: Download template
  ansible.builtin.command: >-
    pveam download {{ pve_lxc_template_storage }}
    {{ pve_lxc_template_latest.stdout }}
  when: pve_lxc_template_latest.stdout not in pve_lxc_template_present.stdout
  changed_when: true

# tofu/stacks/core/runner.tf
resource "proxmox_virtual_environment_container" "gitlab_runner" {
  node_name = var.runner_node
  vm_id     = var.runner_vm_id

  # Unprivileged: container root is not host root
  unprivileged  = true
  start_on_boot = true

  initialization {
    hostname = "runner01"
    ip_config {
      ipv4 {
        address = var.runner_ip
        gateway = var.runner_gateway
      }
    }
    dns {
      servers = [var.runner_dns]
    }
    user_account {
      keys = [trimspace(var.ssh_public_key)]
    }
  }

  cpu {
    cores = 2
  }
  memory {
    dedicated = 2048
    swap      = 512
  }
  disk {
    datastore_id = var.storage_pool
    size         = 20
  }
  network_interface {
    name   = "eth0"
    bridge = "vmbr0"
  }
  operating_system {
    template_file_id = var.runner_template
    type             = "debian"
  }

  # Nesting is required to run containerised CI jobs later
  features {
    nesting = true
  }

  tags = ["ci", "managed-by-tofu"]

  lifecycle {
    # Placement and run state belong to the HA manager, not to the code
    ignore_changes = [node_name, started]
  }
}

Error: unable to create CT 200 - zfs error:
cannot mount 'nvme-vm/subvol-200-disk-0': no mountpoint set

# ansible/roles/gitlab_runner/tasks/main.yml  (extract)
- name: Register runner
  ansible.builtin.command: >-
    gitlab-runner register
    --non-interactive
    --url {{ gitlab_runner_url }}
    --token {{ gitlab_runner_token }}
    --executor {{ gitlab_runner_executor }}
    --shell bash
    --name {{ gitlab_runner_name }}
  when: gitlab_runner_name not in gitlab_runner_list.stderr
  changed_when: true
  no_log: true

- name: Install yq
  ansible.builtin.get_url:
    url: https://github.com/mikefarah/yq/releases/download/v4.44.6/yq_linux_amd64
    dest: /usr/local/bin/yq
    mode: "0755"

- name: Install sops
  ansible.builtin.get_url:
    url: https://github.com/getsops/sops/releases/download/v3.9.4/sops-v3.9.4.linux.amd64
    dest: /usr/local/bin/sops
    mode: "0755"

- name: Install opentofu
  ansible.builtin.shell: |
    set -o pipefail
    curl -fsSL https://get.opentofu.org/install-opentofu.sh -o /tmp/install-tofu.sh
    chmod +x /tmp/install-tofu.sh
    /tmp/install-tofu.sh --install-method standalone --skip-verify
    rm -f /tmp/install-tofu.sh
  args:
    executable: /bin/bash
    creates: /usr/local/bin/tofu

# ansible/gitlab-runner.yml
---
- name: Install and register GitLab runner
  hosts: ci
  become: false
  gather_facts: true
  vars:
    # vars_files does not decrypt SOPS; the lookup plugin does
    kentrowlab_secrets: >-
      {{ lookup('community.sops.sops', playbook_dir + '/../secrets/tofu.sops.yaml') | from_yaml }}
    gitlab_runner_token: "{{ kentrowlab_secrets.gitlab_runner_token }}"
  roles:
    - gitlab_runner

# .gitlab-ci.yml
include:
  - template: Security/Secret-Detection.gitlab-ci.yml

stages:
  - test
  - security
  - plan
  - apply

default:
  tags:
    - kentrowlab

variables:
  TF_STACK: core

secret_detection:
  # Docker-based template: keep it on GitLab.com shared runners
  tags: []

sops-encrypted:
  stage: security
  script:
    - |
      status=0
      files=$(find . -type f \( -name '*.sops.yaml' -o -name '*.sops.yml' \
                                 -o -name '*.sops.json' -o -name '*.sops.env' \) \
                     ! -path './.sops.yaml')
      for f in $files; do
        if grep -q 'ENC\[' "$f"; then echo "OK (encrypted): $f";
        else echo "ERROR (PLAINTEXT): $f"; status=1; fi
      done
      exit $status

.tofu_base:
  before_script:
    - mkdir -p ~/.config/sops/age
    - echo "$SOPS_AGE_KEY" > ~/.config/sops/age/keys.txt
    - chmod 600 ~/.config/sops/age/keys.txt
    - source tofu/env.sh "$TF_STACK"
    - cd "tofu/stacks/$TF_STACK"
    - tofu init -input=false
  after_script:
    - rm -f ~/.config/sops/age/keys.txt

tofu-plan:
  extends: .tofu_base
  stage: plan
  script:
    - tofu plan -input=false -out=tfplan
    - tofu show -no-color tfplan > plan.txt
  artifacts:
    paths:
      - tofu/stacks/$TF_STACK/plan.txt
    expire_in: 1 week
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

tofu-apply:
  extends: .tofu_base
  stage: apply
  script:
    - tofu apply -input=false -auto-approve
  rules:
    # Manual gate: a merge should never silently destroy infrastructure
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
      when: manual
  allow_failure: false

secret_detection job: chosen stage test does not exist

ERROR: Job failed: exit status 127

gitlab-runner 19.2.0 - OpenTofu v1.12.5 - sops 3.9.4 - service active
Pipeline: secret_detection OK  |  sops-encrypted OK  |  tofu-plan OK