Skip to main content

High availability

This guide covers how to deploy Infrahub in a high availability configuration that eliminates single points of failure.

How HA works in Infrahub​

A highly available Infrahub deployment runs redundant instances of every stateful and stateless component, so the system continues to operate even if individual components or availability zones fail:

  • Neo4j cluster — 3+ nodes for graph database redundancy with automatic failover
  • Redis Sentinel — automatic detection of master failures and promotion of replicas
  • RabbitMQ cluster — message bus redundancy with quorum queues replicated across the cluster nodes
  • PostgreSQL replication — task manager database with primary-replica setup
  • Multiple API and worker instances — stateless services distributed across availability zones
  • Redundant load balancers — fronting the API layer for traffic distribution

For the full architecture diagram and deeper explanation, see Scalability and High Availability.

The following sections show how to deploy this architecture using Terraform or Helm.

Deployment examples​

The following examples demonstrate how to deploy Infrahub in a highly available configuration using Kubernetes. These deployments provide resilience, scalability, and are suitable for production environments.

warning

These examples are for reference purposes and may require customization for your specific environment. Ensure you understand the requirements and dependencies before deploying to production.

Using Terraform​

Example HA deployment using Terraform on a 3-node Kubernetes cluster
deploy.tf
terraform {
  required_providers {
    kubectl = {
      source  = "alekc/kubectl"
      version = "2.1.3"
    }
  }
}

provider "helm" {
  kubernetes {
    config_path = "~/.kube/config"
  }
}

provider "kubernetes" {
  config_path = "~/.kube/config"
}

provider "kubectl" {
  config_path = "~/.kube/config"
}

locals {
  target_namespace = "infrahub"
  infrahub_version = "1.2.5"
}

### Infrahub

resource "helm_release" "infrahub_ha" {
  depends_on = [helm_release.taskmanager_ha, helm_release.cache_ha, helm_release.messagequeue_ha, helm_release.database_ha, helm_release.objectstore_ha]

  name    = "infrahub"
  chart   = "oci://registry.opsmill.io/opsmill/chart/infrahub-enterprise"
  version = "3.3.5"

  create_namespace = true
  namespace        = local.target_namespace

  values = [
    <<EOT
infrahub:
  global:
    infrahubTag: ${local.infrahub_version}
  infrahubServer:
    replicas: 3
    persistence:
      enabled: false
    infrahubServer:
      env:
        INFRAHUB_DB_ADDRESS: infrahub-headless
        INFRAHUB_DB_PROTOCOL: neo4j # required for client-side routing
        INFRAHUB_BROKER_ADDRESS: messagequeue-rabbitmq
        INFRAHUB_CACHE_ADDRESS: redis-sentinel-proxy
        INFRAHUB_CACHE_PORT: 6379
        INFRAHUB_WORKFLOW_ADDRESS: prefect-server
        INFRAHUB_WORKFLOW_PORT: 4200
        PREFECT_API_URL: "http://prefect-server:4200/api"

        INFRAHUB_STORAGE_DRIVER: s3
        AWS_ACCESS_KEY_ID: admin
        AWS_SECRET_ACCESS_KEY: password
        AWS_S3_BUCKET_NAME: infrahub-data
        AWS_S3_ENDPOINT_URL: objectstore-minio:9000
        AWS_S3_USE_SSL: "false"

        INFRAHUB_ALLOW_ANONYMOUS_ACCESS: "true"
        INFRAHUB_DB_TYPE: neo4j
        INFRAHUB_LOG_LEVEL: INFO
        INFRAHUB_PRODUCTION: "false"
        INFRAHUB_INITIAL_ADMIN_TOKEN: 06438eb2-8019-4776-878c-0941b1f1d1ec
        INFRAHUB_SECURITY_SECRET_KEY: 327f747f-efac-42be-9e73-999f08f86b92
        INFRAHUB_GIT_REPOSITORIES_DIRECTORY: "/opt/infrahub/git"
    affinity:
      podAntiAffinity:
        requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchLabels:
                service: infrahub-server
            topologyKey: kubernetes.io/hostname
  infrahubTaskWorker:
    replicas: 3
    infrahubTaskWorker:
      env:
        INFRAHUB_DB_ADDRESS: infrahub-headless
        INFRAHUB_DB_PROTOCOL: neo4j # required for client-side routing
        INFRAHUB_BROKER_ADDRESS: messagequeue-rabbitmq
        INFRAHUB_CACHE_ADDRESS: redis-sentinel-proxy
        INFRAHUB_CACHE_PORT: 6379
        INFRAHUB_WORKFLOW_ADDRESS: prefect-server
        INFRAHUB_WORKFLOW_PORT: 4200
        PREFECT_API_URL: "http://prefect-server:4200/api"

        INFRAHUB_STORAGE_DRIVER: s3
        AWS_ACCESS_KEY_ID: admin
        AWS_SECRET_ACCESS_KEY: password
        AWS_S3_BUCKET_NAME: infrahub-data
        AWS_S3_ENDPOINT_URL: objectstore-minio:9000
        AWS_S3_USE_SSL: "false"

        INFRAHUB_DB_TYPE: neo4j
        INFRAHUB_LOG_LEVEL: DEBUG
        INFRAHUB_PRODUCTION: "false"
        INFRAHUB_API_TOKEN: 06438eb2-8019-4776-878c-0941b1f1d1ec
        INFRAHUB_TIMEOUT: "60"
        INFRAHUB_GIT_REPOSITORIES_DIRECTORY: "/opt/infrahub/git"
        PREFECT_WORKER_QUERY_SECONDS: 3
        PREFECT_AGENT_QUERY_INTERVAL: 3
    affinity:
      podAntiAffinity:
        requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchLabels:
                service: infrahub-task-worker
            topologyKey: kubernetes.io/hostname
  redis:
    enabled: false
  neo4j:
    enabled: false
  rabbitmq:
    enabled: false
  prefect-server:
    enabled: false
EOT
  ]
}

#### Infrahub dependencies

resource "helm_release" "database_ha_service" {
  depends_on = [helm_release.database_ha]

  name       = "database-service"
  chart      = "neo4j-headless-service"
  repository = "https://helm.neo4j.com/neo4j/"
  version    = "5.20.0"

  create_namespace = true
  namespace        = local.target_namespace

  values = [
    <<EOT
neo4j:
  name: "infrahub"
EOT
  ]
}

resource "helm_release" "database_ha" {
  count = 3

  name       = "database-${count.index}"
  chart      = "neo4j"
  repository = "https://helm.neo4j.com/neo4j/"
  version    = "5.20.0"

  create_namespace = true
  namespace        = local.target_namespace

  values = [
    <<EOT
neo4j:
  name: "infrahub"
  minimumClusterSize: 3
  resources:
    cpu: "4"
    memory: "8Gi"
  password: "admin"
  edition: "enterprise"
  acceptLicenseAgreement: "yes"
config:
  dbms.security.auth_minimum_password_length: "4"
  dbms.security.procedures.unrestricted: apoc.*
logInitialPassword: false
volumes:
  data:
    mode: "defaultStorageClass"
    defaultStorageClass:
      accessModes:
        - ReadWriteOnce
      requests:
        storage: 10Gi
services:
  neo4j:
    enabled: false
EOT
  ]
}

resource "helm_release" "messagequeue_ha" {
  name    = "messagequeue"
  chart   = "oci://registry-1.docker.io/bitnamicharts/rabbitmq"
  version = "14.4.1"

  create_namespace = true
  namespace        = local.target_namespace

  values = [
    <<EOT
replicaCount: 3
auth:
  username: infrahub
  password: infrahub
metrics:
  enabled: true
startupProbe:
  enabled: true
podAntiAffinityPreset: hard
EOT
  ]
}

resource "helm_release" "objectstore_ha" {
  name    = "objectstore"
  chart   = "oci://registry-1.docker.io/bitnamicharts/minio"
  version = "15.0.5"

  create_namespace = true
  namespace        = local.target_namespace

  values = [
    <<EOT
mode: distributed
statefulset:
  replicaCount: 3
  drivesPerNode: 2
auth:
  rootUser: admin
  rootPassword: password
provisioning:
  enabled: true
  buckets:
    - name: infrahub-data
podAntiAffinityPreset: hard
EOT
  ]
}

#### Task manager

# Workaround since Prefect Helm chart does not use StatefulSets and multiple pod initialization causes issue with concurrent DB init
# StatefulSet would solve this issue because it creates pods sequentially
# Workaround by installing the Helm chart with one replica, and then scale up to 3 replicas
resource "null_resource" "scale_up_taskmanager" {
  depends_on = [helm_release.infrahub_ha]
  provisioner "local-exec" {
    command = "kubectl scale -n ${local.target_namespace} deployment/prefect-server --replicas=3"
  }
}

resource "helm_release" "taskmanager_ha" {
  depends_on = [helm_release.cache_ha, kubectl_manifest.taskmanagerdb_ha]

  name       = "taskmanager"
  chart      = "prefect-server"
  repository = "https://prefecthq.github.io/prefect-helm"
  version    = "2025.2.21193831"

  create_namespace = true
  namespace        = local.target_namespace

  values = [
    <<EOT
global:
  prefect:
    image:
      repository: registry.opsmill.io/opsmill/infrahub-enterprise
      prefectTag: ${local.infrahub_version}
server:
  replicaCount: 1
  command:
    - /usr/bin/tini
    - -g
    - --
  args:
    - uvicorn
    - --host
    - "0.0.0.0"
    - --port
    - "4200"
    - --factory
    - infrahub.prefect_server.app:create_infrahub_prefect
  env:
    - name: PREFECT_UI_SERVE_BASE
      value: /
    - name: PREFECT_MESSAGING_BROKER
      value: prefect_redis.messaging
    - name: PREFECT_MESSAGING_CACHE
      value: prefect_redis.messaging
    - name: PREFECT_REDIS_MESSAGING_HOST
      value: redis-sentinel-proxy
    - name: PREFECT_REDIS_MESSAGING_DB
      value: "1"
secret:
  create: true
  name: ""
  username: "prefect"
  password: "prefect"
  host: "taskmanagerdb-rw"
  port: "5432"
  database: "prefect"
serviceAccount:
  create: false
postgresql:
  enabled: false
EOT
  ]
}

resource "kubernetes_service_v1" "redis_sentinel_proxy_svc" {
  depends_on = [helm_release.cache_ha]

  metadata {
    name      = "redis-sentinel-proxy"
    namespace = local.target_namespace
    labels = {
      "app.kubernetes.io/name" = "redis-sentinel-proxy"
    }
  }

  spec {
    type = "ClusterIP"
    port {
      port        = 6379
      target_port = "redis"
      name        = "cache-ha"
    }
    selector = {
      "app.kubernetes.io/name" = "redis-sentinel-proxy"
    }
  }
}

resource "kubernetes_deployment_v1" "redis_sentinel_proxy_deployment" {
  depends_on = [helm_release.cache_ha]

  metadata {
    name      = "redis-sentinel-proxy"
    namespace = local.target_namespace
    labels = {
      "app.kubernetes.io/name" = "redis-sentinel-proxy"
    }
  }

  spec {
    replicas = 2
    selector {
      match_labels = {
        "app.kubernetes.io/name" = "redis-sentinel-proxy"
      }
    }
    template {
      metadata {
        labels = {
          "app.kubernetes.io/name" = "redis-sentinel-proxy"
        }
      }
      spec {
        affinity {
          pod_anti_affinity {
            required_during_scheduling_ignored_during_execution {
              label_selector {
                match_labels = {
                  "app.kubernetes.io/name" = "redis-sentinel-proxy"
                }
              }
              topology_key = "kubernetes.io/hostname"
            }
          }
        }
        container {
          name  = "redis-sentinel-proxy"
          image = "patrickdk/redis-sentinel-proxy:v1.2"
          args = [
            "-master",
            "mymaster",
            "-listen",
            ":6379",
            "-sentinel",
            "cache:26379",
          ]
          port {
            container_port = 6379
            name           = "redis"
          }
        }
      }
    }
  }
}

resource "helm_release" "cache_ha" {
  name       = "cache"
  chart      = "redis"
  repository = "https://charts.bitnami.com/bitnami"
  version    = "19.5.2"

  create_namespace = true
  namespace        = local.target_namespace

  values = [
    <<EOT
nameOverride: cache
architecture: replication
auth:
  enabled: false
master:
  podAntiAffinityPreset: hard
  persistence:
    enabled: true
  service:
    ports:
      redis: 6379
replicas:
  replicaCount: 3
  podAntiAffinityPreset: hard
sentinel:
  enabled: true
EOT
  ]
}

resource "kubectl_manifest" "taskmanagerdb_ha" {
  depends_on = [helm_release.taskmanagerdb_ha_operator, kubernetes_secret_v1.db_secret]

  yaml_body = <<EOT
apiVersion: postgresql.cnpg.io/v1
kind: Cluster
metadata:
  name: taskmanagerdb
  namespace: ${local.target_namespace}
spec:
  instances: 3
  storage:
    size: 10Gi
  postgresql:
    pg_hba:
      - host all all 10.0.0.0/8 md5
  bootstrap:
    initdb:
      database: prefect
      owner: prefect
      secret:
        name: prefect-user
EOT
}

resource "kubernetes_secret_v1" "db_secret" {
  depends_on = [helm_release.taskmanagerdb_ha_operator]

  metadata {
    name      = "prefect-user"
    namespace = local.target_namespace
  }
  data = {
    username = "prefect"
    password = "prefect"
  }
}

resource "helm_release" "taskmanagerdb_ha_operator" {
  name       = "cnpg"
  chart      = "cloudnative-pg"
  repository = "https://cloudnative-pg.github.io/charts"
  version    = "0.23.2"

  create_namespace = true
  namespace        = local.target_namespace

  values = [
    <<EOT
EOT
  ]
}

Using Helm​

Example HA deployment using Helm and kubectl
redis-values.yaml
---
nameOverride: cache
architecture: replication
auth:
  enabled: false
master:
  podAntiAffinityPreset: hard
  persistence:
    enabled: true
  service:
    ports:
      redis: 6379
replicas:
  replicaCount: 3
  podAntiAffinityPreset: hard
sentinel:
  enabled: true
helm repo add bitnami https://charts.bitnami.com/bitnami
helm repo update
helm install cache bitnami/redis --version 19.5.2 --namespace infrahub --create-namespace --values redis-values.yaml
redis-sentinel-proxy-deploy.yaml
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: redis-sentinel-proxy
  namespace: infrahub
  labels:
    app.kubernetes.io/name: redis-sentinel-proxy
spec:
  replicas: 2
  selector:
    matchLabels:
      app.kubernetes.io/name: redis-sentinel-proxy
  template:
    metadata:
      labels:
        app.kubernetes.io/name: redis-sentinel-proxy
    spec:
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            - labelSelector:
                matchLabels:
                  app.kubernetes.io/name: redis-sentinel-proxy
              topologyKey: kubernetes.io/hostname
      containers:
        - name: redis-sentinel-proxy
          image: patrickdk/redis-sentinel-proxy:v1.2
          args:
            - "-master"
            - "mymaster"
            - "-listen"
            - ":6379"
            - "-sentinel"
            - "cache:26379"
          ports:
            - containerPort: 6379
              name: redis
redis-sentinel-proxy-svc.yaml
---
apiVersion: v1
kind: Service
metadata:
  name: redis-sentinel-proxy
  namespace: infrahub
  labels:
    app.kubernetes.io/name: redis-sentinel-proxy
spec:
  type: ClusterIP
  ports:
    - port: 6379
      targetPort: redis
      name: cache-ha
  selector:
    app.kubernetes.io/name: redis-sentinel-proxy
kubectl apply -f redis-sentinel-proxy-deploy.yaml --namespace infrahub
kubectl apply -f redis-sentinel-proxy-svc.yaml --namespace infrahub
rabbitmq-values.yaml
---
replicaCount: 3
auth:
  username: infrahub
  password: infrahub
metrics:
  enabled: true
startupProbe:
  enabled: true
podAntiAffinityPreset: hard
helm install messagequeue oci://registry-1.docker.io/bitnamicharts/rabbitmq --version 14.4.1 --namespace infrahub --create-namespace --values rabbitmq-values.yaml
neo4j-values.yaml
---
neo4j:
  name: "infrahub"
  minimumClusterSize: 3
  resources:
    cpu: "4"
    memory: "8Gi"
  password: "admin"
  edition: "enterprise"
  acceptLicenseAgreement: "yes"
config:
  dbms.security.auth_minimum_password_length: "4"
  dbms.security.procedures.unrestricted: apoc.*
logInitialPassword: false
volumes:
  data:
    mode: "defaultStorageClass"
    defaultStorageClass:
      accessModes:
        - ReadWriteOnce
      requests:
        storage: 10Gi
services:
  neo4j:
    enabled: false
helm repo add neo4j https://helm.neo4j.com/neo4j/
helm repo update
kubectl create secret generic neo4j-user --namespace infrahub --from-literal=NEO4J_AUTH=neo4j/admin
helm install database-0 neo4j/neo4j --version 2026.5.0 --namespace infrahub --values neo4j-values.yaml
helm install database-1 neo4j/neo4j --version 2026.5.0 --namespace infrahub --values neo4j-values.yaml
helm install database-2 neo4j/neo4j --version 2026.5.0 --namespace infrahub --values neo4j-values.yaml

Once all instances are running, install the headless service:

service-values.yaml
---
neo4j:
  name: "infrahub"
helm install database-service neo4j/neo4j-headless-service --version 2026.5.0 --namespace infrahub --values service-values.yaml
minio-values.yaml
---
mode: distributed
statefulset:
  replicaCount: 3
  drivesPerNode: 2
auth:
  rootUser: admin
  rootPassword: password
provisioning:
  enabled: true
  buckets:
    - name: infrahub-data
podAntiAffinityPreset: hard
helm install objectstore oci://registry-1.docker.io/bitnamicharts/minio --version 15.0.5 --namespace infrahub --create-namespace --values minio-values.yaml
helm repo add cnpg https://cloudnative-pg.github.io/charts
helm repo update
helm install cnpg cnpg/cloudnative-pg --version 0.23.2 --namespace infrahub --create-namespace
kubectl create secret generic prefect-user --namespace infrahub --from-literal=username=prefect --from-literal=password=prefect
taskmanagerdb.yaml
---
apiVersion: postgresql.cnpg.io/v1
kind: Cluster
metadata:
  name: taskmanagerdb
  namespace: infrahub
spec:
  instances: 3
  storage:
    size: 10Gi
  postgresql:
    pg_hba:
      - host all all 10.0.0.0/8 md5
  bootstrap:
    initdb:
      database: prefect
      owner: prefect
      secret:
        name: prefect-user
kubectl apply -f taskmanagerdb.yaml --namespace infrahub
infrahub-values.yaml
---
infrahub:
  global:
    infrahubTag: 1.2.5
  infrahubServer:
    replicas: 3
    persistence:
      enabled: false
    infrahubServer:
      env:
        INFRAHUB_DB_ADDRESS: infrahub-headless
        INFRAHUB_DB_PROTOCOL: neo4j # required for client-side routing
        INFRAHUB_BROKER_ADDRESS: messagequeue-rabbitmq
        INFRAHUB_CACHE_ADDRESS: redis-sentinel-proxy
        INFRAHUB_CACHE_PORT: 6379
        INFRAHUB_WORKFLOW_ADDRESS: prefect-server
        INFRAHUB_WORKFLOW_PORT: 4200
        PREFECT_API_URL: "http://prefect-server:4200/api"
        INFRAHUB_STORAGE_DRIVER: s3
        AWS_ACCESS_KEY_ID: admin
        AWS_SECRET_ACCESS_KEY: password
        AWS_S3_BUCKET_NAME: infrahub-data
        AWS_S3_ENDPOINT_URL: objectstore-minio:9000
        AWS_S3_USE_SSL: "false"
        INFRAHUB_ALLOW_ANONYMOUS_ACCESS: "true"
        INFRAHUB_DB_TYPE: neo4j
        INFRAHUB_LOG_LEVEL: INFO
        INFRAHUB_PRODUCTION: "false"
        INFRAHUB_INITIAL_ADMIN_TOKEN: 06438eb2-8019-4776-878c-0941b1f1d1ec
        INFRAHUB_SECURITY_SECRET_KEY: 327f747f-efac-42be-9e73-999f08f86b92
        INFRAHUB_GIT_REPOSITORIES_DIRECTORY: "/opt/infrahub/git"
    affinity:
      podAntiAffinity:
        requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchLabels:
                service: infrahub-server
            topologyKey: kubernetes.io/hostname
  infrahubTaskWorker:
    replicas: 3
    infrahubTaskWorker:
      env:
        INFRAHUB_DB_ADDRESS: infrahub-headless
        INFRAHUB_DB_PROTOCOL: neo4j # required for client-side routing
        INFRAHUB_BROKER_ADDRESS: messagequeue-rabbitmq
        INFRAHUB_CACHE_ADDRESS: redis-sentinel-proxy
        INFRAHUB_CACHE_PORT: 6379
        INFRAHUB_WORKFLOW_ADDRESS: prefect-server
        INFRAHUB_WORKFLOW_PORT: 4200
        PREFECT_API_URL: "http://prefect-server:4200/api"
        INFRAHUB_STORAGE_DRIVER: s3
        AWS_ACCESS_KEY_ID: admin
        AWS_SECRET_ACCESS_KEY: password
        AWS_S3_BUCKET_NAME: infrahub-data
        AWS_S3_ENDPOINT_URL: objectstore-minio:9000
        AWS_S3_USE_SSL: "false"
        INFRAHUB_DB_TYPE: neo4j
        INFRAHUB_LOG_LEVEL: DEBUG
        INFRAHUB_PRODUCTION: "false"
        INFRAHUB_API_TOKEN: 06438eb2-8019-4776-878c-0941b1f1d1ec
        INFRAHUB_TIMEOUT: "60"
        INFRAHUB_GIT_REPOSITORIES_DIRECTORY: "/opt/infrahub/git"
        PREFECT_WORKER_QUERY_SECONDS: 3
        PREFECT_AGENT_QUERY_INTERVAL: 3
    affinity:
      podAntiAffinity:
        requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchLabels:
                service: infrahub-task-worker
            topologyKey: kubernetes.io/hostname
  redis:
    enabled: false
  neo4j:
    enabled: false
  rabbitmq:
    enabled: false
  prefect-server:
    enabled: false
helm install infrahub oci://registry.opsmill.io/opsmill/chart/infrahub-enterprise --version 4.2.4-small --namespace infrahub --create-namespace --values infrahub-values.yaml

Connecting Infrahub to a Neo4j cluster​

When running Neo4j as a cluster, set INFRAHUB_DB_PROTOCOL=neo4j so the driver discovers the cluster topology and routes reads and writes to the appropriate members. With the default bolt protocol, all traffic pins to a single member without routing awareness.

Infrahub needs at least one reachable cluster member to bootstrap this routing discovery, which can be provided in two ways:

  • DNS routing: Point INFRAHUB_DB_ADDRESS at a single DNS name that resolves to all cluster members, such as the Kubernetes headless service used in the Helm example above (infrahub-headless.infrahub.svc.cluster.local).
  • Member list: Set INFRAHUB_DB_ADDRESS to a comma-separated list of cluster members in host[:port] format. Members without an explicit port use INFRAHUB_DB_PORT. This is useful when no single DNS name covers all members, for example with virtual machine based deployments.
INFRAHUB_DB_PROTOCOL=neo4j
INFRAHUB_DB_ADDRESS=neo4j-core-1,neo4j-core-2,neo4j-core-3

In both cases the driver tries the members in turn until one answers, so the initial connection succeeds even when some members are down.

Connecting Infrahub to a RabbitMQ cluster​

RabbitMQ 4.0 removed classic queue mirroring. A classic queue is stored on one node of the cluster: when that node stops, the queue is unavailable, and Infrahub cannot send or receive the messages that use it until the node is back. A quorum queue stores a copy of its messages on several nodes, and RabbitMQ elects a new leader for the queue when the node of the current leader stops.

Set INFRAHUB_BROKER_QUEUE_TYPE to quorum on the Infrahub server and the task workers, so that Infrahub declares its durable queues as quorum queues:

INFRAHUB_BROKER_QUEUE_TYPE=quorum
  • The queues that each Infrahub process creates for its own replies and events stay classic queues. RabbitMQ deletes them when the process disconnects, and the process creates them again when it reconnects.
  • A quorum queue stays available while a majority of its copies are on running nodes, so a 3-node cluster continues to operate when one node stops.
  • Set INFRAHUB_BROKER_ADDRESS to an address that reaches every node of the cluster, such as the Kubernetes service of the RabbitMQ cluster or a TCP load balancer. When the node of a connection stops, Infrahub reconnects to that address.
warning

Do not set the default queue type of the RabbitMQ virtual host to quorum while INFRAHUB_BROKER_QUEUE_TYPE is classic. RabbitMQ then declares the durable queues of Infrahub as quorum queues with an argument that quorum queues reject, and the Infrahub server fails to start.

Switch an existing deployment to quorum queues​

RabbitMQ does not change the type of an existing queue. When the durable queues already exist as classic queues, the Infrahub server fails to start with a PRECONDITION_FAILED - inequivalent arg error until you delete them. Deleting a queue discards the messages it holds.

  1. Stop the Infrahub server and the task workers.

  2. Delete the durable queues in the virtual host that Infrahub uses (INFRAHUB_BROKER_VIRTUALHOST, / by default). The queue names start with the value of INFRAHUB_BROKER_NAMESPACE, infrahub by default:

    rabbitmqctl delete_queue --vhost / infrahub.rpcs
    rabbitmqctl delete_queue --vhost / infrahub.delay.five_seconds
    rabbitmqctl delete_queue --vhost / infrahub.delay.ten_seconds
    rabbitmqctl delete_queue --vhost / infrahub.delay.twenty_seconds

  3. Set INFRAHUB_BROKER_QUEUE_TYPE to quorum on the Infrahub server and the task workers.

  4. Start the Infrahub server, then the task workers. The server declares the durable queues again as quorum queues.

To switch back to classic queues, follow the same steps with INFRAHUB_BROKER_QUEUE_TYPE set to classic.

Load balancer configuration​

For high availability deployments, a load balancer is useful to distribute traffic across multiple Infrahub API server instances. The load balancer must be configured to properly handle both HTTP/HTTPS and WebSocket connections.

HAProxy configuration example
global
    maxconn 4096
    log stdout local0

defaults
    mode http
    timeout connect 5000ms
    timeout client 50000ms
    timeout server 50000ms
    option httplog

# Frontend for Infrahub API
frontend infrahub_frontend
    bind *:80
    bind *:443 ssl crt /etc/ssl/certs/infrahub.pem

    # Redirect HTTP to HTTPS
    redirect scheme https if !{ ssl_fc }

    # ACL for WebSocket connections
    acl is_websocket hdr(Upgrade) -i WebSocket
    acl is_websocket_path path_beg /ws /graphql-ws

    # Use WebSocket backend for WS connections
    use_backend infrahub_websocket if is_websocket || is_websocket_path

    # Default to HTTP backend
    default_backend infrahub_http

# Backend for regular HTTP/HTTPS traffic
backend infrahub_http
    balance roundrobin
    option httpchk GET /api/health
    http-check expect status 200

    # Sticky sessions for GraphQL subscriptions
    cookie SERVERID insert indirect nocache

    server infrahub-api-1 10.0.1.10:8000 check cookie api1
    server infrahub-api-2 10.0.1.11:8000 check cookie api2
    server infrahub-api-3 10.0.1.12:8000 check cookie api3

# Backend for WebSocket connections
backend infrahub_websocket
    balance source
    option http-server-close
    option forceclose

    # WebSocket specific health check
    option httpchk GET /api/health
    http-check expect status 200

    server infrahub-api-1 10.0.1.10:8000 check
    server infrahub-api-2 10.0.1.11:8000 check
    server infrahub-api-3 10.0.1.12:8000 check
NGINX configuration example
upstream infrahub_backend {
    # IP hash for session affinity
    ip_hash;

    server 10.0.1.10:8000 max_fails=3 fail_timeout=30s;
    server 10.0.1.11:8000 max_fails=3 fail_timeout=30s;
    server 10.0.1.12:8000 max_fails=3 fail_timeout=30s;
}

server {
    listen 80;
    server_name infrahub.example.com;

    # Redirect HTTP to HTTPS
    return 301 https://$server_name$request_uri;
}

server {
    listen 443 ssl http2;
    server_name infrahub.example.com;

    ssl_certificate /etc/ssl/certs/infrahub.pem;
    ssl_certificate_key /etc/ssl/private/infrahub.key;
    ssl_protocols TLSv1.2 TLSv1.3;
    ssl_ciphers HIGH:!aNULL:!MD5;

    # Client body size for large GraphQL queries
    client_max_body_size 10M;

    # Timeouts for long-running operations
    proxy_connect_timeout 600;
    proxy_send_timeout 600;
    proxy_read_timeout 600;
    send_timeout 600;

    # Health check endpoint
    location /health {
        access_log off;
        proxy_pass http://infrahub_backend/api/health;
    }

    # WebSocket support for GraphQL subscriptions
    location ~ ^/(ws|graphql-ws) {
        proxy_pass http://infrahub_backend;
        proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection "upgrade";
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;

        # Disable buffering for WebSocket
        proxy_buffering off;
    }

    # Regular API traffic
    location / {
        proxy_pass http://infrahub_backend;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;

        # Enable keepalive
        proxy_http_version 1.1;
        proxy_set_header Connection "";
    }
}