diff --git a/README.md b/README.md index d7ea344a1..d314a9774 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,7 @@ certain backends. Some of them can be disabled at compile-time using a build tag |----------|:-----------|:--------------------| | Kubernetes exporter | Kubernetes exporter reports node problems to Kubernetes API server: temporary problems get reported as Events, and permanent problems get reported as Node Conditions. | | Prometheus exporter | Prometheus exporter reports node problems and metrics locally as Prometheus metrics | +| HTTP exporter | HTTP exporter serves the local `/healthz`, `/conditions` and `/debug/pprof` endpoints. It keeps node conditions in memory and does not require a Kubernetes API server, so it also works when `--enable-k8s-exporter` is `false`. | | [Stackdriver exporter](https://github.com/kubernetes/node-problem-detector/blob/master/config/exporter/stackdriver-exporter.json) | Stackdriver exporter reports node problems and metrics to Stackdriver Monitoring API. | disable_stackdriver_exporter # Usage @@ -122,8 +123,14 @@ For example, to run without auth, use the following config: http://APISERVER_IP:APISERVER_PORT?inClusterConfig=false ``` Refer to [heapster docs](https://github.com/kubernetes/heapster/blob/master/docs/source-configuration.md#kubernetes) for a complete list of available options. -* `--address`: The address to bind the node problem detector server. -* `--port`: The port to bind the node problem detector server. Use 0 to disable. + +#### For HTTP exporter + +The HTTP exporter serves `/healthz`, `/conditions` and `/debug/pprof`. It does not talk to the +Kubernetes API server, so these endpoints are available even when `--enable-k8s-exporter` is `false`. + +* `--address`: The address to bind the node problem detector server, default to `127.0.0.1`. +* `--port`: The port to bind the node problem detector server, default to 20256. Use 0 to disable. #### For Prometheus exporter diff --git a/cmd/nodeproblemdetector/node_problem_detector.go b/cmd/nodeproblemdetector/node_problem_detector.go index 41648b758..5fb8e0ee0 100644 --- a/cmd/nodeproblemdetector/node_problem_detector.go +++ b/cmd/nodeproblemdetector/node_problem_detector.go @@ -25,6 +25,7 @@ import ( _ "k8s.io/node-problem-detector/cmd/nodeproblemdetector/problemdaemonplugins" "k8s.io/node-problem-detector/cmd/options" "k8s.io/node-problem-detector/pkg/exporters" + "k8s.io/node-problem-detector/pkg/exporters/httpexporter" "k8s.io/node-problem-detector/pkg/exporters/k8sexporter" "k8s.io/node-problem-detector/pkg/exporters/prometheusexporter" "k8s.io/node-problem-detector/pkg/problemdaemon" @@ -51,6 +52,10 @@ func npdMain(ctx context.Context, npdo *options.NodeProblemDetectorOptions) erro // Initialize exporters. defaultExporters := []types.Exporter{} + if he := httpexporter.NewExporterOrDie(npdo); he != nil { + defaultExporters = append(defaultExporters, he) + klog.Info("HTTP exporter started.") + } if ke := k8sexporter.NewExporterOrDie(ctx, npdo); ke != nil { defaultExporters = append(defaultExporters, ke) klog.Info("K8s exporter started.") diff --git a/pkg/exporters/httpexporter/http_exporter.go b/pkg/exporters/httpexporter/http_exporter.go new file mode 100644 index 000000000..83a43bc0b --- /dev/null +++ b/pkg/exporters/httpexporter/http_exporter.go @@ -0,0 +1,104 @@ +/* +Copyright 2026 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package httpexporter + +import ( + "net" + "net/http" + "net/http/pprof" + "strconv" + "sync" + + "k8s.io/klog/v2" + + "k8s.io/node-problem-detector/cmd/options" + "k8s.io/node-problem-detector/pkg/types" + "k8s.io/node-problem-detector/pkg/util" +) + +type httpExporter struct { + mu sync.RWMutex + conditions map[string]types.Condition +} + +// NewExporterOrDie creates the standalone HTTP exporter and starts the server. +// Returns nil if --port is 0 (disabled). Panics on bind errors. +func NewExporterOrDie(npdo *options.NodeProblemDetectorOptions) types.Exporter { + if npdo.ServerPort <= 0 { + return nil + } + + he := &httpExporter{ + conditions: make(map[string]types.Condition), + } + + addr := net.JoinHostPort(npdo.ServerAddress, strconv.Itoa(npdo.ServerPort)) + mux := he.buildMux() + go func() { + if err := http.ListenAndServe(addr, mux); err != nil { + klog.Fatalf("Failed to start HTTP server: %v", err) + } + }() + + klog.Infof("HTTP exporter started on %s", addr) + return he +} + +func (he *httpExporter) buildMux() *http.ServeMux { + mux := http.NewServeMux() + + // Add healthz http request handler. Always return ok now, add more health check + // logic in the future. + mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + if _, err := w.Write([]byte("ok")); err != nil { + klog.Errorf("Failed to write response: %v", err) + } + }) + + // Add the handler to serve condition http request. + mux.HandleFunc("/conditions", func(w http.ResponseWriter, r *http.Request) { + util.ReturnHTTPJson(w, he.getConditions()) + }) + + // register pprof + mux.HandleFunc("/debug/pprof/", pprof.Index) + mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) + mux.HandleFunc("/debug/pprof/profile", pprof.Profile) + mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + mux.HandleFunc("/debug/pprof/trace", pprof.Trace) + + return mux +} + +func (he *httpExporter) ExportProblems(status *types.Status) { + he.mu.Lock() + defer he.mu.Unlock() + for _, cdt := range status.Conditions { + he.conditions[cdt.Type] = cdt + } +} + +func (he *httpExporter) getConditions() []types.Condition { + he.mu.RLock() + defer he.mu.RUnlock() + conditions := make([]types.Condition, 0, len(he.conditions)) + for _, c := range he.conditions { + conditions = append(conditions, c) + } + return conditions +} diff --git a/pkg/exporters/httpexporter/http_exporter_test.go b/pkg/exporters/httpexporter/http_exporter_test.go new file mode 100644 index 000000000..1f5c912fd --- /dev/null +++ b/pkg/exporters/httpexporter/http_exporter_test.go @@ -0,0 +1,256 @@ +/* +Copyright 2026 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package httpexporter + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "k8s.io/node-problem-detector/cmd/options" + "k8s.io/node-problem-detector/pkg/types" +) + +func newTestExporter() *httpExporter { + return &httpExporter{conditions: make(map[string]types.Condition)} +} + +func newTestCondition(conditionType, reason string, status types.ConditionStatus) types.Condition { + return types.Condition{ + Type: conditionType, + Status: status, + Transition: time.Now(), + Reason: reason, + Message: "test message", + } +} + +func TestNewExporterOrDieDisabled(t *testing.T) { + testCases := []struct { + name string + port int + }{ + { + name: "Zero port disables the exporter", + port: 0, + }, + { + name: "Negative port disables the exporter", + port: -1, + }, + } + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + npdo := &options.NodeProblemDetectorOptions{ + ServerPort: test.port, + ServerAddress: "127.0.0.1", + } + assert.Nil(t, NewExporterOrDie(npdo)) + }) + } +} + +func TestExportProblems(t *testing.T) { + // The exporter is shared across cases so that each case builds on the + // state left behind by the previous one. + he := newTestExporter() + + testCases := []struct { + name string + // status is exported at the start of the case. + status *types.Status + // wantReasons is the full expected condition store afterwards, keyed by + // condition type. + wantReasons map[string]string + }{ + { + name: "First condition is stored", + status: &types.Status{ + Source: "test", + Conditions: []types.Condition{newTestCondition("TypeA", "ReasonA", types.True)}, + }, + wantReasons: map[string]string{"TypeA": "ReasonA"}, + }, + { + name: "Condition of a new type is stored alongside the existing one", + status: &types.Status{ + Source: "test", + Conditions: []types.Condition{newTestCondition("TypeB", "ReasonB", types.True)}, + }, + wantReasons: map[string]string{"TypeA": "ReasonA", "TypeB": "ReasonB"}, + }, + { + name: "Condition of a known type overrides instead of appending", + status: &types.Status{ + Source: "test", + Conditions: []types.Condition{newTestCondition("TypeA", "ReasonAUpdated", types.False)}, + }, + wantReasons: map[string]string{"TypeA": "ReasonAUpdated", "TypeB": "ReasonB"}, + }, + { + name: "Multiple conditions in one status are all stored", + status: &types.Status{ + Source: "test", + Conditions: []types.Condition{ + newTestCondition("TypeC", "ReasonC", types.True), + newTestCondition("TypeD", "ReasonD", types.True), + }, + }, + wantReasons: map[string]string{ + "TypeA": "ReasonAUpdated", "TypeB": "ReasonB", + "TypeC": "ReasonC", "TypeD": "ReasonD", + }, + }, + { + name: "Events do not affect the condition store", + status: &types.Status{ + Source: "test", + Events: []types.Event{{ + Severity: types.Warn, + Timestamp: time.Now(), + Reason: "TestEvent", + Message: "test event message", + }}, + }, + wantReasons: map[string]string{ + "TypeA": "ReasonAUpdated", "TypeB": "ReasonB", + "TypeC": "ReasonC", "TypeD": "ReasonD", + }, + }, + } + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + he.ExportProblems(test.status) + + gotReasons := map[string]string{} + for _, c := range he.getConditions() { + gotReasons[c.Type] = c.Reason + } + assert.Equal(t, test.wantReasons, gotReasons) + }) + } +} + +func TestHandlers(t *testing.T) { + testCases := []struct { + name string + // seed is exported before the request is served. + seed []types.Condition + path string + wantStatus int + wantContentType string + // wantBody is compared exactly when set. + wantBody string + }{ + { + name: "healthz always reports ok", + path: "/healthz", + wantStatus: http.StatusOK, + wantBody: "ok", + }, + { + name: "conditions serves an empty array when nothing was exported", + path: "/conditions", + wantStatus: http.StatusOK, + wantContentType: "application/json", + wantBody: "[]", + }, + { + name: "pprof index is registered", + path: "/debug/pprof/", + wantStatus: http.StatusOK, + }, + } + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + he := newTestExporter() + if len(test.seed) != 0 { + he.ExportProblems(&types.Status{Source: "test", Conditions: test.seed}) + } + + w := httptest.NewRecorder() + req := httptest.NewRequestWithContext(context.Background(), http.MethodGet, test.path, nil) + he.buildMux().ServeHTTP(w, req) + + assert.Equal(t, test.wantStatus, w.Code) + if test.wantContentType != "" { + assert.Equal(t, test.wantContentType, w.Header().Get("Content-type")) + } + if test.wantBody != "" { + assert.Equal(t, test.wantBody, w.Body.String()) + } + }) + } +} + +func TestConditionsHandlerServesExportedCondition(t *testing.T) { + he := newTestExporter() + he.ExportProblems(&types.Status{ + Source: "test", + Conditions: []types.Condition{newTestCondition("KernelDeadlock", "AUFSUmountHung", types.True)}, + }) + + w := httptest.NewRecorder() + req := httptest.NewRequestWithContext(context.Background(), http.MethodGet, "/conditions", nil) + he.buildMux().ServeHTTP(w, req) + + var conditions []types.Condition + assert.NoError(t, json.Unmarshal(w.Body.Bytes(), &conditions)) + assert.Len(t, conditions, 1) + assert.Equal(t, "KernelDeadlock", conditions[0].Type) + assert.Equal(t, "AUFSUmountHung", conditions[0].Reason) + assert.Equal(t, types.True, conditions[0].Status) +} + +func TestConcurrentAccess(t *testing.T) { + he := newTestExporter() + const goroutines = 10 + const iterations = 100 + + var wg sync.WaitGroup + for i := range goroutines { + wg.Add(2) + go func(i int) { + defer wg.Done() + for range iterations { + he.ExportProblems(&types.Status{ + Source: "test", + Conditions: []types.Condition{newTestCondition(fmt.Sprintf("Type%d", i), "Reason", types.True)}, + }) + } + }(i) + go func() { + defer wg.Done() + for range iterations { + he.getConditions() + } + }() + } + wg.Wait() + + assert.Len(t, he.getConditions(), goroutines) +} diff --git a/pkg/exporters/k8sexporter/k8s_exporter.go b/pkg/exporters/k8sexporter/k8s_exporter.go index d47e04ff0..f0551a28e 100644 --- a/pkg/exporters/k8sexporter/k8s_exporter.go +++ b/pkg/exporters/k8sexporter/k8s_exporter.go @@ -18,10 +18,6 @@ package k8sexporter import ( "context" - "net" - "net/http" - "net/http/pprof" - "strconv" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog/v2" @@ -65,7 +61,6 @@ func NewExporterOrDie(ctx context.Context, npdo *options.NodeProblemDetectorOpti updateConditions: npdo.K8sExporterUpdateNodeConditions, } - ke.startHTTPReporting(npdo) ke.conditionManager.Start(ctx) return &ke @@ -84,42 +79,6 @@ func (ke *k8sExporter) ExportProblems(status *types.Status) { } } -func (ke *k8sExporter) startHTTPReporting(npdo *options.NodeProblemDetectorOptions) { - if npdo.ServerPort <= 0 { - return - } - mux := http.NewServeMux() - - // Add healthz http request handler. Always return ok now, add more health check - // logic in the future. - mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - if _, err := w.Write([]byte("ok")); err != nil { - klog.Errorf("Failed to write response: %v", err) - } - }) - - // Add the handler to serve condition http request. - mux.HandleFunc("/conditions", func(w http.ResponseWriter, r *http.Request) { - util.ReturnHTTPJson(w, ke.conditionManager.GetConditions()) - }) - - // register pprof - mux.HandleFunc("/debug/pprof/", pprof.Index) - mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) - mux.HandleFunc("/debug/pprof/profile", pprof.Profile) - mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) - mux.HandleFunc("/debug/pprof/trace", pprof.Trace) - - addr := net.JoinHostPort(npdo.ServerAddress, strconv.Itoa(npdo.ServerPort)) - go func() { - err := http.ListenAndServe(addr, mux) - if err != nil { - klog.Fatalf("Failed to start server: %v", err) - } - }() -} - func waitForAPIServerReadyWithTimeout(ctx context.Context, c problemclient.Client, npdo *options.NodeProblemDetectorOptions) error { return wait.PollUntilContextTimeout(ctx, npdo.APIServerWaitInterval, npdo.APIServerWaitTimeout, true, func(ctx context.Context) (done bool, err error) { // If NPD can get the node object from kube-apiserver, the server is