Envoy router verifies ate apiserver's serving cert

cmd/ateapi/main.go

@@ -25,10 +25,10 @@ import (
 	"time"
 
 	"github.com/agent-substrate/substrate/cmd/ateapi/internal/controlapi"
-	"github.com/agent-substrate/substrate/cmd/ateapi/internal/credbundle"
 	"github.com/agent-substrate/substrate/cmd/ateapi/internal/sessionidentity"
 	"github.com/agent-substrate/substrate/cmd/ateapi/internal/store/ateredis"
 	"github.com/agent-substrate/substrate/internal/ateinterceptors"
+	"github.com/agent-substrate/substrate/internal/credbundle"
 	"github.com/agent-substrate/substrate/internal/serverboot"
 	"github.com/agent-substrate/substrate/internal/version"
 	"github.com/agent-substrate/substrate/pkg/client/clientset/versioned"

cmd/atenet/internal/root.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/agent-substrate/substrate/cmd/atenet/internal/router"
 	"github.com/agent-substrate/substrate/internal/version"
 	"github.com/spf13/cobra"
 )
@@ -37,6 +38,6 @@ func Execute() {
 }
 
 func init() {
-	rootCmd.AddCommand(NewRouterCmd())
+	rootCmd.AddCommand(router.NewRouterCmd())
 	rootCmd.AddCommand(NewDnsCmd())
 }

cmd/atenet/internal/router.go → cmd/atenet/internal/router/config.go

@@ -12,25 +12,49 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package internal
+package router
 
 import (
 	"fmt"
 	"time"
 
 	"github.com/spf13/cobra"
-
-	"github.com/agent-substrate/substrate/cmd/atenet/internal/router"
 )
 
+type authConfig struct {
+	AteapiClientCertPath string
+	AteapiCACertsPath    string
+}
+
+// routerConfig holds deployment setup and endpoint options for the router node instance.
+type routerConfig struct {
+	Standalone     bool
+	Namespace      string
+	Kubeconfig     string
+	AteapiAddr     string
+	HttpPort       int
+	XdsPort        int
+	ExtprocPort    int
+	ExtprocAddr    string
+	EnvoyImage     string
+	TemplatesFile  string
+	StatusPort     int
+	HealthInterval time.Duration
+	HttpsPort      int
+	EnvoyCertPath  string
+	LogLevel       string
+	MetricsAddr    string
+	Auth           authConfig
+}
+
 func NewRouterCmd() *cobra.Command {
-	var cfg router.RouterConfig
+	var cfg routerConfig
 
 	cmd := &cobra.Command{
 		Use:   "router",
 		Short: "Router components including xDS server and Envoy ExtProc gateway processing server",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			srv, err := router.NewRouterServer(cfg)
+			srv, err := NewRouterServer(cfg)
 			if err != nil {
 				return fmt.Errorf("failed to create router server: %w", err)
 			}
@@ -46,6 +70,8 @@ func NewRouterCmd() *cobra.Command {
 	cmd.Flags().StringVar(&cfg.Namespace, "namespace", "default", "Target operations namespace")
 	cmd.Flags().StringVar(&cfg.Kubeconfig, "kubeconfig", "", "Absolute path to the kubeconfig configuration file")
 	cmd.Flags().StringVar(&cfg.AteapiAddr, "ateapi-address", "api.ate-system.svc:443", "gRPC host address of the cluster ateapi Control instance")
+	cmd.Flags().StringVar(&cfg.Auth.AteapiClientCertPath, "ateapi-client-cert", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "Path to the podidentity credential bundle the router presents as its client cert to ateapi.")
+	cmd.Flags().StringVar(&cfg.Auth.AteapiCACertsPath, "ateapi-ca-certs", "/run/servicedns-ca.podcert.ate.dev/trust-bundle.pem", "Path to the servicedns trust bundle used to verify ateapi's serving cert.")
 	cmd.Flags().IntVar(&cfg.HttpPort, "port-http", 8080, "TCP port for workload traffic entering through the Envoy Router")
 	cmd.Flags().IntVar(&cfg.XdsPort, "port-xds", 18000, "TCP port listening for the xDS dynamic Envoy connections")
 	cmd.Flags().IntVar(&cfg.ExtprocPort, "port-extproc", 50051, "Listen port for the Envoy dynamic External Processing (ext_proc) server")
@@ -55,7 +81,7 @@ func NewRouterCmd() *cobra.Command {
 	cmd.Flags().IntVar(&cfg.StatusPort, "status-port", 4040, "Port to serve /statusz on (set <= 0 to disable serving status)")
 	cmd.Flags().DurationVar(&cfg.HealthInterval, "health-interval", 1*time.Second, "Interval for checking health of dependent services")
 	cmd.Flags().IntVar(&cfg.HttpsPort, "port-https", 8443, "TCP port for HTTPS workload traffic entering through the Envoy Router")
-	cmd.Flags().StringVar(&cfg.EnvoyCertPath, "envoy-cert-path", "", "Path to the Envoy certificate file (if empty, a self-signed cert will be generated for testing)")
+	cmd.Flags().StringVar(&cfg.EnvoyCertPath, "envoy-cert-path", "", "Path to the Envoy certificate file.")
 
 	return cmd
 }

cmd/atenet/internal/router/controller.go

@@ -28,7 +28,7 @@ import (
 type Controller struct {
 	k8sClient  client.Client
 	clientset  kubernetes.Interface
-	cfg        RouterConfig
+	cfg        routerConfig
 	xdsSrv     *XdsServer
 	extprocSrv *ExtProcServer
 
@@ -39,7 +39,7 @@ type Controller struct {
 func NewController(
 	k8sClient client.Client,
 	clientset kubernetes.Interface,
-	cfg RouterConfig,
+	cfg routerConfig,
 	xdsSrv *XdsServer,
 	extprocSrv *ExtProcServer,
 ) *Controller {

cmd/atenet/internal/router/envoyrunner.go

@@ -37,10 +37,10 @@ const (
 // Envoy proxy instance running inside Kubernetes.
 type envoyrunner struct {
 	k8sClient client.Client
-	cfg       RouterConfig
+	cfg       routerConfig
 }
 
-func newEnvoyRunner(k8sClient client.Client, cfg RouterConfig) *envoyrunner {
+func newEnvoyRunner(k8sClient client.Client, cfg routerConfig) *envoyrunner {
 	return &envoyrunner{
 		k8sClient: k8sClient,
 		cfg:       cfg,

cmd/atenet/internal/router/health.go

@@ -53,10 +53,10 @@ type routerHealth struct {
 	interval  time.Duration
 	clientset kubernetes.Interface
 	apiClient ateapipb.ControlClient
-	cfg       RouterConfig
+	cfg       routerConfig
 }
 
-func newRouterHealth(interval time.Duration, clientset kubernetes.Interface, apiClient ateapipb.ControlClient, cfg RouterConfig) *routerHealth {
+func newRouterHealth(interval time.Duration, clientset kubernetes.Interface, apiClient ateapipb.ControlClient, cfg routerConfig) *routerHealth {
 	if interval <= 0 {
 		interval = time.Second
 	}

cmd/atenet/internal/router/router.go

@@ -16,23 +16,17 @@ package router
 
 import (
 	"context"
-	"crypto/rand"
-	"crypto/rsa"
 	"crypto/tls"
 	"crypto/x509"
-	"crypto/x509/pkix"
-	"encoding/pem"
 	"errors"
 	"fmt"
 	"log/slog"
-	"math/big"
 	"net"
 	"net/http"
 	"os"
 	"os/signal"
 	"strings"
 	"syscall"
-	"time"
 
 	"github.com/spf13/cobra"
 	"golang.org/x/sync/errgroup"
@@ -46,6 +40,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
 
+	"github.com/agent-substrate/substrate/internal/credbundle"
 	"github.com/agent-substrate/substrate/internal/serverboot"
 	v1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1"
 	"github.com/agent-substrate/substrate/pkg/proto/ateapipb"
@@ -60,29 +55,9 @@ func init() {
 	utilruntime.Must(v1alpha1.AddToScheme(scheme))
 }
 
-// RouterConfig holds deployment setup and endpoint options for the router node instance.
-type RouterConfig struct {
-	Standalone     bool
-	Namespace      string
-	Kubeconfig     string
-	AteapiAddr     string
-	HttpPort       int
-	XdsPort        int
-	ExtprocPort    int
-	ExtprocAddr    string
-	EnvoyImage     string
-	TemplatesFile  string
-	StatusPort     int
-	HealthInterval time.Duration
-	HttpsPort      int
-	EnvoyCertPath  string
-	LogLevel       string
-	MetricsAddr    string
-}
-
 // RouterServer instantiates and coordinates runtime threads executing system modules.
 type RouterServer struct {
-	cfg RouterConfig
+	cfg routerConfig
 
 	Cmd        *cobra.Command
 	k8sClient  client.Client
@@ -93,7 +68,7 @@ type RouterServer struct {
 	atStore    atStore
 }
 
-func NewRouterServer(cfg RouterConfig) (*RouterServer, error) {
+func NewRouterServer(cfg routerConfig) (*RouterServer, error) {
 	var k8sClient client.Client
 	var clientset kubernetes.Interface
 	var err error
@@ -125,7 +100,12 @@ func NewRouterServer(cfg RouterConfig) (*RouterServer, error) {
 		}
 	}
 
-	conn, err := grpc.NewClient(cfg.AteapiAddr, grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{InsecureSkipVerify: true})))
+	creds, err := cfg.apiTransportCredentials()
+	if err != nil {
+		return nil, fmt.Errorf("failed to build ateapi transport credentials: %w", err)
+	}
+
+	conn, err := grpc.NewClient(cfg.AteapiAddr, grpc.WithTransportCredentials(creds))
 	if err != nil {
 		return nil, fmt.Errorf("failed to establish grpc channel to ateapi client: %w", err)
 	}
@@ -149,6 +129,46 @@ func NewRouterServer(cfg RouterConfig) (*RouterServer, error) {
 	}, nil
 }
 
+// ateapiTransportCreds builds the TLS credentials the router uses to dial
+// ateapi. When both the servicedns trust bundle and the podidentity client
+// credential bundle are present (the in-cluster case, mounted via projected
+// pod-certificate volumes), it performs mTLS: it verifies ateapi's serving cert
+// against the servicedns trust bundle and presents its own podidentity SPIFFE
+// client cert. When that material is absent, it returns an error rather than
+// falling back to an insecure connection.
+func (cfg routerConfig) apiTransportCredentials() (credentials.TransportCredentials, error) {
+	tlsCfg, err := apiTLSConfig(cfg)
+	if err != nil {
+		return nil, err
+	}
+	return credentials.NewTLS(tlsCfg), nil
+}
+
+func apiTLSConfig(cfg routerConfig) (*tls.Config, error) {
+	if _, err := os.Stat(cfg.Auth.AteapiCACertsPath); err != nil {
+		return nil, fmt.Errorf("error reading ate apiserver CA path from %q, error=%w",
+			cfg.Auth.AteapiCACertsPath, err)
+	}
+	if _, err := os.Stat(cfg.Auth.AteapiClientCertPath); err != nil {
+		return nil, fmt.Errorf("error reading ate apiserver client cert path from %q, error=%w",
+			cfg.Auth.AteapiClientCertPath, err)
+	}
+
+	caBytes, err := os.ReadFile(cfg.Auth.AteapiCACertsPath)
+	if err != nil {
+		return nil, fmt.Errorf("read ateapi CA certs: %w", err)
+	}
+	rootCAs := x509.NewCertPool()
+	if !rootCAs.AppendCertsFromPEM(caBytes) {
+		return nil, fmt.Errorf("parse ateapi CA certs from %s", cfg.Auth.AteapiCACertsPath)
+	}
+
+	return &tls.Config{
+		RootCAs:              rootCAs,
+		GetClientCertificate: credbundle.ClientLoader(cfg.Auth.AteapiClientCertPath),
+	}, nil
+}
+
 func (s *RouterServer) Run(ctx context.Context) error {
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
@@ -188,17 +208,7 @@ func (s *RouterServer) Run(ctx context.Context) error {
 	xdsSrv := NewXdsServer(s.cfg.XdsPort)
 	xdsSrv.SetConfig(s.cfg.HttpPort, s.cfg.ExtprocPort, s.cfg.ExtprocAddr)
 
-	var certContent, keyContent string
-	if s.cfg.EnvoyCertPath == "" {
-		slog.InfoContext(ctx, "No Envoy certificate path provided, generating self-signed certificate for testing")
-		var err error
-		certContent, keyContent, err = generateSelfSignedCert()
-		if err != nil {
-			return fmt.Errorf("failed to generate self-signed cert: %w", err)
-		}
-	}
-
-	xdsSrv.SetTlsConfig(s.cfg.HttpsPort, s.cfg.EnvoyCertPath, certContent, keyContent)
+	xdsSrv.SetTlsConfig(s.cfg.HttpsPort, s.cfg.EnvoyCertPath)
 	if s.extprocSrv == nil {
 		routeDuration, err := newRouteDurationHistogram()
 		if err != nil {
@@ -274,39 +284,3 @@ func (s *RouterServer) Run(ctx context.Context) error {
 
 	return g.Wait()
 }
-
-func generateSelfSignedCert() (string, string, error) {
-	priv, err := rsa.GenerateKey(rand.Reader, 2048)
-	if err != nil {
-		return "", "", err
-	}
-
-	template := x509.Certificate{
-		SerialNumber: big.NewInt(1),
-		Subject: pkix.Name{
-			Organization: []string{"Substrate Local Test"},
-		},
-		NotBefore: time.Now(),
-		NotAfter:  time.Now().Add(time.Hour * 24 * 365),
-
-		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
-		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
-		BasicConstraintsValid: true,
-		DNSNames:              []string{"localhost"},
-	}
-
-	derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)
-	if err != nil {
-		return "", "", err
-	}
-
-	certPem := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
-
-	privBytes, err := x509.MarshalPKCS8PrivateKey(priv)
-	if err != nil {
-		return "", "", err
-	}
-	keyPem := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: privBytes})
-
-	return string(certPem), string(keyPem), nil
-}