Securing a NATS Cluster with cfssl

Secure NATS Cluster in Kubernetes using the NATS Operator

Features

  • Clients TLS setup

  • TLS based auth certs via secret

    • Reloading supported by only updating secret

  • Routes TLS setup

  • Advertising public IP per NATS server for external access

Creating the Certificates

Generating the Root CA Certs

{
"CN": "nats.io",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"OU": "nats.io"
}
]
}
(
cd certs
# CA certs
cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
)

Setup the profiles for the Root CA, we will have 3 main profiles: one for the clients connecting, one for the servers, and another one for the full mesh routing connections between the servers.

{
"signing": {
"default": {
"expiry": "43800h"
},
"profiles": {
"server": {
"expiry": "43800h",
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
]
},
"client": {
"expiry": "43800h",
"usages": [
"signing",
"key encipherment",
"client auth"
]
},
"route": {
"expiry": "43800h",
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
]
}
}
}
}

Generating the NATS server certs

First we generate the certificates for the server.

{
"CN": "nats.io",
"hosts": [
"localhost",
"*.nats-cluster.default.svc",
"*.nats-cluster-mgmt.default.svc",
"nats-cluster",
"nats-cluster-mgmt",
"nats-cluster.default.svc",
"nats-cluster-mgmt.default.svc",
"nats-cluster.default.svc.cluster.local",
"nats-cluster-mgmt.default.svc.cluster.local",
"*.nats-cluster.default.svc.cluster.local",
"*.nats-cluster-mgmt.default.svc.cluster.local"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"OU": "Operator"
}
]
}
(
# Generating the peer certificates
cd certs
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server server.json | cfssljson -bare server
)

Generating the NATS server routes certs

We will also be setting up TLS for the full mesh routes.

{
"CN": "nats.io",
"hosts": [
"localhost",
"*.nats-cluster.default.svc",
"*.nats-cluster-mgmt.default.svc",
"nats-cluster",
"nats-cluster-mgmt",
"nats-cluster.default.svc",
"nats-cluster-mgmt.default.svc",
"nats-cluster.default.svc.cluster.local",
"nats-cluster-mgmt.default.svc.cluster.local",
"*.nats-cluster.default.svc.cluster.local",
"*.nats-cluster-mgmt.default.svc.cluster.local"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"OU": "Operator"
}
]
}
# Generating the peer certificates
(
cd certs
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=route route.json | cfssljson -bare route
)

Generating the certs for the clients (CNCF && ACME)

{
"CN": "nats.io",
"hosts": [""],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"OU": "CNCF"
}
]
}
(
cd certs
# Generating NATS client certs
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client client.json | cfssljson -bare client
)

Kubectl Create

cd certs kubectl create secret generic nats-tls-example --from-file=ca.pem --from-file=server-key.pem --from-file=server.pem kubectl create secret generic nats-tls-routes-example --from-file=ca.pem --from-file=route-key.pem --from-file=route.pem kubectl create secret generic nats-tls-client-example --from-file=ca.pem --from-file=client-key.pem --from-file=client.pem

Create the Auth secret

{
"users": [
{ "username": "CN=nats.io,OU=ACME" },
{ "username": "CN=nats.io,OU=CNCF",
"permissions": {
"publish": ["hello.*"],
"subscribe": ["hello.world"]
}
}
],
"default_permissions": {
"publish": ["SANDBOX.*"],
"subscribe": ["PUBLIC.>"]
}
}
kubectl create secret generic nats-tls-users --from-file=users.json

Create a cluster with TLS

echo '
apiVersion: "nats.io/v1alpha2"
kind: "NatsCluster"
metadata:
name: "nats-cluster"
spec:
size: 3
# Using custom edge nats server image for TLS verify and map support.
serverImage: "wallyqs/nats-server"
version: "edge-2.0.0-RC5"
tls:
enableHttps: true
# Certificates to secure the NATS client connections:
serverSecret: "nats-tls-example"
# Certificates to secure the routes.
routesSecret: "nats-tls-routes-example"
auth:
tlsVerifyAndMap: true
clientsAuthSecret: "nats-tls-users"
# How long to wait for authentication
clientsAuthTimeout: 5
pod:
# To be able to reload the secret changes
enableConfigReload: true
reloaderImage: connecteverything/nats-server-config-reloader
# Bind the port 4222 as the host port to allow external access.
enableClientsHostPort: true
# Initializer container that resolves the external IP from the
# container where it is running.
advertiseExternalIP: true
# Image of container that resolves external IP from K8S API
bootconfigImage: "wallyqs/nats-boot-config"
bootconfigImageTag: "0.5.0"
# Service account required to be able to find the external IP
template:
spec:
serviceAccountName: "nats-server"
' | kubectl apply -f -

Create APP using certs

Adding a new pod which uses the certificates

Development

package main
import (
"encoding/json"
"flag"
"fmt"
"log"
"time"
"github.com/nats-io/go-nats"
"github.com/nats-io/nuid"
)
func main() {
var (
serverList string
rootCACertFile string
clientCertFile string
clientKeyFile string
)
flag.StringVar(&serverList, "s", "tls://nats-1.nats-cluster.default.svc:4222", "List of NATS of servers available")
flag.StringVar(&rootCACertFile, "cacert", "./certs/ca.pem", "Root CA Certificate File")
flag.StringVar(&clientCertFile, "cert", "./certs/client.pem", "Client Certificate File")
flag.StringVar(&clientKeyFile, "key", "./certs/client-key.pem", "Client Private key")
flag.Parse()
log.Println("NATS endpoint:", serverList)
log.Println("Root CA:", rootCACertFile)
log.Println("Client Cert:", clientCertFile)
log.Println("Client Key:", clientKeyFile)
// Connect options
rootCA := nats.RootCAs(rootCACertFile)
clientCert := nats.ClientCert(clientCertFile, clientKeyFile)
alwaysReconnect := nats.MaxReconnects(-1)
var nc *nats.Conn
var err error
for {
nc, err = nats.Connect(serverList, rootCA, clientCert, alwaysReconnect)
if err != nil {
log.Printf("Error while connecting to NATS, backing off for a sec... (error: %s)", err)
time.Sleep(1 * time.Second)
continue
}
break
}
nc.Subscribe("discovery.*.status", func(m *nats.Msg) {
log.Printf("[Received on %q] %s", m.Subject, string(m.Data))
})
discoverySubject := fmt.Sprintf("discovery.%s.status", nuid.Next())
info := struct {
InMsgs uint64 `json:"in_msgs"`
OutMsgs uint64 `json:"out_msgs"`
Reconnects uint64 `json:"reconnects"`
CurrentServer string `json:"current_server"`
Servers []string `json:"servers"`
}{}
for range time.NewTicker(1 * time.Second).C {
stats := nc.Stats()
info.InMsgs = stats.InMsgs
info.OutMsgs = stats.OutMsgs
info.Reconnects = stats.Reconnects
info.CurrentServer = nc.ConnectedUrl()
info.Servers = nc.Servers()
payload, err := json.Marshal(info)
if err != nil {
log.Printf("Error marshalling data: %s", err)
}
err = nc.Publish(discoverySubject, payload)
if err != nil {
log.Printf("Error during publishing: %s", err)
}
nc.Flush()
}
}
FROM golang:1.11.0-alpine3.8 AS builder
COPY . /go/src/github.com/nats-io/nats-kubernetes/examples/nats-cluster-routes-tls/app
WORKDIR /go/src/github.com/nats-io/nats-kubernetes/examples/nats-cluster-routes-tls/app
RUN apk add --update git
RUN go get -u github.com/nats-io/go-nats
RUN go get -u github.com/nats-io/nuid
RUN CGO_ENABLED=0 go build -o nats-client-app -v -a ./client.go
FROM scratch
COPY --from=builder /go/src/github.com/nats-io/nats-kubernetes/examples/nats-cluster-routes-tls/app/nats-client-app /nats-client-app
ENTRYPOINT ["/nats-client-app"]
docker build . -t wallyqs/nats-client-app
docker run wallyqs/nats-client-app
docker push wallyqs/nats-client-app

Pod spec

echo ' apiVersion: apps/v1beta2 kind: Deployment
## The name of the deployment
metadata: name: nats-client-app
spec:
## This selector has to match the template.metadata.labels section
## which is below in the PodSpec
selector: matchLabels: name: nats-client-app
## Number of instances
replicas: 1
## PodSpec
template: metadata: labels: name: nats-client-app spec: volumes:
* name: "client-tls-certs"
secret:
secretName: "nats-tls-client-example"
containers:
* name: nats-client-app
command: \["/nats-client-app", "-s", "tls://nats-cluster.default.svc:4222", "-cacert", '/etc/nats-client-tls-certs/ca.pem', '-cert', '/etc/nats-client-tls-certs/client.pem', '-key', '/etc/nats-client-tls-certs/client-key.pem'\]
image: wallyqs/nats-client-app:latest
imagePullPolicy: Always
volumeMounts:
* name: "client-tls-certs"
mountPath: "/etc/nats-client-tls-certs/"
' \| kubectl apply -f -