Securing a NATS Cluster with cfssl

Secure NATS Cluster in Kubernetes using the NATS Operator
Features
Clients TLS setup
TLS based auth certs via secret
Reloading supported by only updating secret
Routes TLS setup
Advertising public IP per NATS server for external access
Creating the Certificates
Generating the Root CA Certs
{
    "CN": "nats.io",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "OU": "nats.io"
        }
    ]
}
(
  cd certs
​
  # CA certs
  cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
)
Setup the profiles for the Root CA, we will have 3 main profiles: one for the clients connecting, one for the servers, and another one for the full mesh routing connections between the servers.
{
    "signing": {
        "default": {
            "expiry": "43800h"
        },
        "profiles": {
            "server": {
                "expiry": "43800h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "server auth",
                    "client auth"
                ]
            },
            "client": {
                "expiry": "43800h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "client auth"
                ]
            },
            "route": {
                "expiry": "43800h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "server auth",
                    "client auth"
                ]
            }
        }
    }
}
Generating the NATS server certs
First we generate the certificates for the server.
{
    "CN": "nats.io",
    "hosts": [
        "localhost",
        "*.nats-cluster.default.svc",
        "*.nats-cluster-mgmt.default.svc",
        "nats-cluster",
        "nats-cluster-mgmt",
        "nats-cluster.default.svc",
        "nats-cluster-mgmt.default.svc",
        "nats-cluster.default.svc.cluster.local",
        "nats-cluster-mgmt.default.svc.cluster.local",
        "*.nats-cluster.default.svc.cluster.local",
        "*.nats-cluster-mgmt.default.svc.cluster.local"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "OU": "Operator"
        }
    ]
}
(
  # Generating the peer certificates
  cd certs
  cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server server.json | cfssljson -bare server
)
Generating the NATS server routes certs
We will also be setting up TLS for the full mesh routes.
{
    "CN": "nats.io",
    "hosts": [
        "localhost",
        "*.nats-cluster.default.svc",
        "*.nats-cluster-mgmt.default.svc",
        "nats-cluster",
        "nats-cluster-mgmt",
        "nats-cluster.default.svc",
        "nats-cluster-mgmt.default.svc",
        "nats-cluster.default.svc.cluster.local",
        "nats-cluster-mgmt.default.svc.cluster.local",
        "*.nats-cluster.default.svc.cluster.local",
        "*.nats-cluster-mgmt.default.svc.cluster.local"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "OU": "Operator"
        }
    ]
}
# Generating the peer certificates
(
  cd certs
  cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=route route.json | cfssljson -bare route
)
Generating the certs for the clients (CNCF && ACME)
{
    "CN": "nats.io",
    "hosts": [""],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "OU": "CNCF"
        }
    ]
}
(
  cd certs
  # Generating NATS client certs
  cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client client.json | cfssljson -bare client
)
Kubectl Create
cd certs kubectl create secret generic nats-tls-example --from-file=ca.pem --from-file=server-key.pem --from-file=server.pem kubectl create secret generic nats-tls-routes-example --from-file=ca.pem --from-file=route-key.pem --from-file=route.pem kubectl create secret generic nats-tls-client-example --from-file=ca.pem --from-file=client-key.pem --from-file=client.pem
Create the Auth secret
{
  "users": [
    { "username": "CN=nats.io,OU=ACME" },
    { "username": "CN=nats.io,OU=CNCF",
      "permissions": {
    "publish": ["hello.*"],
    "subscribe": ["hello.world"]
      }
    }
  ],
  "default_permissions": {
    "publish": ["SANDBOX.*"],
    "subscribe": ["PUBLIC.>"]
  }
}
kubectl create secret generic nats-tls-users --from-file=users.json
Create a cluster with TLS
echo '
apiVersion: "nats.io/v1alpha2"
kind: "NatsCluster"
metadata:
  name: "nats-cluster"
spec:
  size: 3
​
  # Using custom edge nats server image for TLS verify and map support.
  serverImage: "wallyqs/nats-server"
  version: "edge-2.0.0-RC5"
​
  tls:
    enableHttps: true
​
    # Certificates to secure the NATS client connections:
    serverSecret: "nats-tls-example"
​
    # Certificates to secure the routes.
    routesSecret: "nats-tls-routes-example"
​
  auth:
    tlsVerifyAndMap: true
    clientsAuthSecret: "nats-tls-users"
​
    # How long to wait for authentication
    clientsAuthTimeout: 5
​
  pod:
    # To be able to reload the secret changes
    enableConfigReload: true
    reloaderImage: connecteverything/nats-server-config-reloader
​
    # Bind the port 4222 as the host port to allow external access.
    enableClientsHostPort: true
​
    # Initializer container that resolves the external IP from the
    # container where it is running.
    advertiseExternalIP: true
​
    # Image of container that resolves external IP from K8S API
    bootconfigImage: "wallyqs/nats-boot-config"
    bootconfigImageTag: "0.5.0"
​
  # Service account required to be able to find the external IP
  template:
    spec:
      serviceAccountName: "nats-server"
' | kubectl apply -f -
Create APP using certs
Adding a new pod which uses the certificates
Development
package main
​
import (
    "encoding/json"
    "flag"
    "fmt"
    "log"
    "time"
​
    "github.com/nats-io/go-nats"
    "github.com/nats-io/nuid"
)
​
func main() {
    var (
        serverList     string
        rootCACertFile string
        clientCertFile string
        clientKeyFile  string
    )
    flag.StringVar(&serverList, "s", "tls://nats-1.nats-cluster.default.svc:4222", "List of NATS of servers available")
    flag.StringVar(&rootCACertFile, "cacert", "./certs/ca.pem", "Root CA Certificate File")
    flag.StringVar(&clientCertFile, "cert", "./certs/client.pem", "Client Certificate File")
    flag.StringVar(&clientKeyFile, "key", "./certs/client-key.pem", "Client Private key")
    flag.Parse()
​
    log.Println("NATS endpoint:", serverList)
    log.Println("Root CA:", rootCACertFile)
    log.Println("Client Cert:", clientCertFile)
    log.Println("Client Key:", clientKeyFile)
​
    // Connect options
    rootCA := nats.RootCAs(rootCACertFile)
    clientCert := nats.ClientCert(clientCertFile, clientKeyFile)
    alwaysReconnect := nats.MaxReconnects(-1)
​
    var nc *nats.Conn
    var err error
    for {
        nc, err = nats.Connect(serverList, rootCA, clientCert, alwaysReconnect)
        if err != nil {
            log.Printf("Error while connecting to NATS, backing off for a sec... (error: %s)", err)
            time.Sleep(1 * time.Second)
            continue
        }
        break
    }
​
    nc.Subscribe("discovery.*.status", func(m *nats.Msg) {
        log.Printf("[Received on %q] %s", m.Subject, string(m.Data))
    })
​
    discoverySubject := fmt.Sprintf("discovery.%s.status", nuid.Next())
    info := struct {
        InMsgs        uint64   `json:"in_msgs"`
        OutMsgs       uint64   `json:"out_msgs"`
        Reconnects    uint64   `json:"reconnects"`
        CurrentServer string   `json:"current_server"`
        Servers       []string `json:"servers"`
    }{}
​
    for range time.NewTicker(1 * time.Second).C {
        stats := nc.Stats()
        info.InMsgs = stats.InMsgs
        info.OutMsgs = stats.OutMsgs
        info.Reconnects = stats.Reconnects
        info.CurrentServer = nc.ConnectedUrl()
        info.Servers = nc.Servers()
        payload, err := json.Marshal(info)
        if err != nil {
            log.Printf("Error marshalling data: %s", err)
        }
        err = nc.Publish(discoverySubject, payload)
        if err != nil {
            log.Printf("Error during publishing: %s", err)
        }
        nc.Flush()
    }
}
FROM golang:1.11.0-alpine3.8 AS builder
COPY . /go/src/github.com/nats-io/nats-kubernetes/examples/nats-cluster-routes-tls/app
WORKDIR /go/src/github.com/nats-io/nats-kubernetes/examples/nats-cluster-routes-tls/app
RUN apk add --update git
RUN go get -u github.com/nats-io/go-nats
RUN go get -u github.com/nats-io/nuid
RUN CGO_ENABLED=0 go build -o nats-client-app -v -a ./client.go
​
FROM scratch
COPY --from=builder /go/src/github.com/nats-io/nats-kubernetes/examples/nats-cluster-routes-tls/app/nats-client-app /nats-client-app
ENTRYPOINT ["/nats-client-app"]
docker build . -t wallyqs/nats-client-app
docker run wallyqs/nats-client-app
docker push wallyqs/nats-client-app
Pod spec
echo ' apiVersion: apps/v1beta2 kind: Deployment
​
## The name of the deployment
​
metadata: name: nats-client-app
​
spec:
​
## This selector has to match the template.metadata.labels section
​
## which is below in the PodSpec
​
selector: matchLabels: name: nats-client-app
​
## Number of instances
​
replicas: 1
​
## PodSpec
​
template: metadata: labels: name: nats-client-app spec: volumes:
​
* name: "client-tls-certs"
​
  secret:
​
    secretName: "nats-tls-client-example"
​
  containers:
​
* name: nats-client-app
​
  command: \["/nats-client-app", "-s", "tls://nats-cluster.default.svc:4222", "-cacert", '/etc/nats-client-tls-certs/ca.pem', '-cert', '/etc/nats-client-tls-certs/client.pem', '-key', '/etc/nats-client-tls-certs/client-key.pem'\]
​
  image: wallyqs/nats-client-app:latest
​
  imagePullPolicy: Always
​
  volumeMounts:
​
  * name: "client-tls-certs"
​
    mountPath: "/etc/nats-client-tls-certs/"
​
    ' \| kubectl apply -f -
NATS on Kubernetes - Previous
NATS Cluster and Cert Manager
Last updated 4 weeks ago