Securing a NATS Cluster with cfssl

Secure NATS Cluster in Kubernetes using the NATS Operator

Features

  • Clients TLS setup
  • TLS based auth certs via secret
    • Reloading supported by only updating secret
  • Routes TLS setup
  • Advertising public IP per NATS server for external access

Creating the Certificates

Generating the Root CA Certs

1
{
2
"CN": "nats.io",
3
"key": {
4
"algo": "rsa",
5
"size": 2048
6
},
7
"names": [
8
{
9
"OU": "nats.io"
10
}
11
]
12
}
Copied!
1
(
2
cd certs
3
4
# CA certs
5
cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
6
)
Copied!
Setup the profiles for the Root CA, we will have 3 main profiles: one for the clients connecting, one for the servers, and another one for the full mesh routing connections between the servers.
1
{
2
"signing": {
3
"default": {
4
"expiry": "43800h"
5
},
6
"profiles": {
7
"server": {
8
"expiry": "43800h",
9
"usages": [
10
"signing",
11
"key encipherment",
12
"server auth",
13
"client auth"
14
]
15
},
16
"client": {
17
"expiry": "43800h",
18
"usages": [
19
"signing",
20
"key encipherment",
21
"client auth"
22
]
23
},
24
"route": {
25
"expiry": "43800h",
26
"usages": [
27
"signing",
28
"key encipherment",
29
"server auth",
30
"client auth"
31
]
32
}
33
}
34
}
35
}
Copied!

Generating the NATS server certs

First we generate the certificates for the server.
1
{
2
"CN": "nats.io",
3
"hosts": [
4
"localhost",
5
"*.nats-cluster.default.svc",
6
"*.nats-cluster-mgmt.default.svc",
7
"nats-cluster",
8
"nats-cluster-mgmt",
9
"nats-cluster.default.svc",
10
"nats-cluster-mgmt.default.svc",
11
"nats-cluster.default.svc.cluster.local",
12
"nats-cluster-mgmt.default.svc.cluster.local",
13
"*.nats-cluster.default.svc.cluster.local",
14
"*.nats-cluster-mgmt.default.svc.cluster.local"
15
],
16
"key": {
17
"algo": "rsa",
18
"size": 2048
19
},
20
"names": [
21
{
22
"OU": "Operator"
23
}
24
]
25
}
Copied!
1
(
2
# Generating the peer certificates
3
cd certs
4
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server server.json | cfssljson -bare server
5
)
Copied!

Generating the NATS server routes certs

We will also be setting up TLS for the full mesh routes.
1
{
2
"CN": "nats.io",
3
"hosts": [
4
"localhost",
5
"*.nats-cluster.default.svc",
6
"*.nats-cluster-mgmt.default.svc",
7
"nats-cluster",
8
"nats-cluster-mgmt",
9
"nats-cluster.default.svc",
10
"nats-cluster-mgmt.default.svc",
11
"nats-cluster.default.svc.cluster.local",
12
"nats-cluster-mgmt.default.svc.cluster.local",
13
"*.nats-cluster.default.svc.cluster.local",
14
"*.nats-cluster-mgmt.default.svc.cluster.local"
15
],
16
"key": {
17
"algo": "rsa",
18
"size": 2048
19
},
20
"names": [
21
{
22
"OU": "Operator"
23
}
24
]
25
}
Copied!
1
# Generating the peer certificates
2
(
3
cd certs
4
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=route route.json | cfssljson -bare route
5
)
Copied!

Generating the certs for the clients (CNCF && ACME)

1
{
2
"CN": "nats.io",
3
"hosts": [""],
4
"key": {
5
"algo": "rsa",
6
"size": 2048
7
},
8
"names": [
9
{
10
"OU": "CNCF"
11
}
12
]
13
}
Copied!
1
(
2
cd certs
3
# Generating NATS client certs
4
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client client.json | cfssljson -bare client
5
)
Copied!

Kubectl Create

1
cd certs kubectl create secret generic nats-tls-example --from-file=ca.pem --from-file=server-key.pem --from-file=server.pem kubectl create secret generic nats-tls-routes-example --from-file=ca.pem --from-file=route-key.pem --from-file=route.pem kubectl create secret generic nats-tls-client-example --from-file=ca.pem --from-file=client-key.pem --from-file=client.pem
Copied!

Create the Auth secret

1
{
2
"users": [
3
{ "username": "CN=nats.io,OU=ACME" },
4
{ "username": "CN=nats.io,OU=CNCF",
5
"permissions": {
6
"publish": ["hello.*"],
7
"subscribe": ["hello.world"]
8
}
9
}
10
],
11
"default_permissions": {
12
"publish": ["SANDBOX.*"],
13
"subscribe": ["PUBLIC.>"]
14
}
15
}
Copied!
1
kubectl create secret generic nats-tls-users --from-file=users.json
Copied!

Create a cluster with TLS

1
echo '
2
apiVersion: "nats.io/v1alpha2"
3
kind: "NatsCluster"
4
metadata:
5
name: "nats-cluster"
6
spec:
7
size: 3
8
9
# Using custom edge nats server image for TLS verify and map support.
10
serverImage: "wallyqs/nats-server"
11
version: "edge-2.0.0-RC5"
12
13
tls:
14
enableHttps: true
15
16
# Certificates to secure the NATS client connections:
17
serverSecret: "nats-tls-example"
18
19
# Certificates to secure the routes.
20
routesSecret: "nats-tls-routes-example"
21
22
auth:
23
tlsVerifyAndMap: true
24
clientsAuthSecret: "nats-tls-users"
25
26
# How long to wait for authentication
27
clientsAuthTimeout: 5
28
29
pod:
30
# To be able to reload the secret changes
31
enableConfigReload: true
32
reloaderImage: connecteverything/nats-server-config-reloader
33
34
# Bind the port 4222 as the host port to allow external access.
35
enableClientsHostPort: true
36
37
# Initializer container that resolves the external IP from the
38
# container where it is running.
39
advertiseExternalIP: true
40
41
# Image of container that resolves external IP from K8S API
42
bootconfigImage: "wallyqs/nats-boot-config"
43
bootconfigImageTag: "0.5.0"
44
45
# Service account required to be able to find the external IP
46
template:
47
spec:
48
serviceAccountName: "nats-server"
49
' | kubectl apply -f -
Copied!

Create APP using certs

Adding a new pod which uses the certificates

Development
1
package main
2
3
import (
4
"encoding/json"
5
"flag"
6
"fmt"
7
"log"
8
"time"
9
10
"github.com/nats-io/go-nats"
11
"github.com/nats-io/nuid"
12
)
13
14
func main() {
15
var (
16
serverList string
17
rootCACertFile string
18
clientCertFile string
19
clientKeyFile string
20
)
21
flag.StringVar(&serverList, "s", "tls://nats-1.nats-cluster.default.svc:4222", "List of NATS of servers available")
22
flag.StringVar(&rootCACertFile, "cacert", "./certs/ca.pem", "Root CA Certificate File")
23
flag.StringVar(&clientCertFile, "cert", "./certs/client.pem", "Client Certificate File")
24
flag.StringVar(&clientKeyFile, "key", "./certs/client-key.pem", "Client Private key")
25
flag.Parse()
26
27
log.Println("NATS endpoint:", serverList)
28
log.Println("Root CA:", rootCACertFile)
29
log.Println("Client Cert:", clientCertFile)
30
log.Println("Client Key:", clientKeyFile)
31
32
// Connect options
33
rootCA := nats.RootCAs(rootCACertFile)
34
clientCert := nats.ClientCert(clientCertFile, clientKeyFile)
35
alwaysReconnect := nats.MaxReconnects(-1)
36
37
var nc *nats.Conn
38
var err error
39
for {
40
nc, err = nats.Connect(serverList, rootCA, clientCert, alwaysReconnect)
41
if err != nil {
42
log.Printf("Error while connecting to NATS, backing off for a sec... (error: %s)", err)
43
time.Sleep(1 * time.Second)
44
continue
45
}
46
break
47
}
48
49
nc.Subscribe("discovery.*.status", func(m *nats.Msg) {
50
log.Printf("[Received on %q] %s", m.Subject, string(m.Data))
51
})
52
53
discoverySubject := fmt.Sprintf("discovery.%s.status", nuid.Next())
54
info := struct {
55
InMsgs uint64 `json:"in_msgs"`
56
OutMsgs uint64 `json:"out_msgs"`
57
Reconnects uint64 `json:"reconnects"`
58
CurrentServer string `json:"current_server"`
59
Servers []string `json:"servers"`
60
}{}
61
62
for range time.NewTicker(1 * time.Second).C {
63
stats := nc.Stats()
64
info.InMsgs = stats.InMsgs
65
info.OutMsgs = stats.OutMsgs
66
info.Reconnects = stats.Reconnects
67
info.CurrentServer = nc.ConnectedUrl()
68
info.Servers = nc.Servers()
69
payload, err := json.Marshal(info)
70
if err != nil {
71
log.Printf("Error marshalling data: %s", err)
72
}
73
err = nc.Publish(discoverySubject, payload)
74
if err != nil {
75
log.Printf("Error during publishing: %s", err)
76
}
77
nc.Flush()
78
}
79
}
Copied!
1
FROM golang:1.11.0-alpine3.8 AS builder
2
COPY . /go/src/github.com/nats-io/nats-kubernetes/examples/nats-cluster-routes-tls/app
3
WORKDIR /go/src/github.com/nats-io/nats-kubernetes/examples/nats-cluster-routes-tls/app
4
RUN apk add --update git
5
RUN go get -u github.com/nats-io/go-nats
6
RUN go get -u github.com/nats-io/nuid
7
RUN CGO_ENABLED=0 go build -o nats-client-app -v -a ./client.go
8
9
FROM scratch
10
COPY --from=builder /go/src/github.com/nats-io/nats-kubernetes/examples/nats-cluster-routes-tls/app/nats-client-app /nats-client-app
11
ENTRYPOINT ["/nats-client-app"]
Copied!
1
docker build . -t wallyqs/nats-client-app
2
docker run wallyqs/nats-client-app
3
docker push wallyqs/nats-client-app
Copied!

Pod spec

1
echo ' apiVersion: apps/v1beta2 kind: Deployment
2
3
## The name of the deployment
4
5
metadata: name: nats-client-app
6
7
spec:
8
9
## This selector has to match the template.metadata.labels section
10
11
## which is below in the PodSpec
12
13
selector: matchLabels: name: nats-client-app
14
15
## Number of instances
16
17
replicas: 1
18
19
## PodSpec
20
21
template: metadata: labels: name: nats-client-app spec: volumes:
22
23
* name: "client-tls-certs"
24
25
secret:
26
27
secretName: "nats-tls-client-example"
28
29
containers:
30
31
* name: nats-client-app
32
33
command: \["/nats-client-app", "-s", "tls://nats-cluster.default.svc:4222", "-cacert", '/etc/nats-client-tls-certs/ca.pem', '-cert', '/etc/nats-client-tls-certs/client.pem', '-key', '/etc/nats-client-tls-certs/client-key.pem'\]
34
35
image: wallyqs/nats-client-app:latest
36
37
imagePullPolicy: Always
38
39
volumeMounts:
40
41
* name: "client-tls-certs"
42
43
mountPath: "/etc/nats-client-tls-certs/"
44
45
' \| kubectl apply -f -
Copied!