From 3b98f638809b9d7434b1fac4fa1e422e87b15f25 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Thu, 3 Sep 2026 14:47:11 +0200 Subject: [PATCH] feat(provider): get-service-config, addhost, and endpoint conventions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Providers could not see the definition of the service they manage, nor make their resource addressable from consuming services. - A provider may emit {"type": "get-service-config"} on stdout; compose answers on the provider's stdin with one JSON line holding the resolved canonical configuration of the provider's own service, straight from the in-memory model. Detection is by construction: a compose that predates the message aborts on it and never writes to stdin, so the provider treats EOF as 'unsupported, upgrade compose'. - A provider may emit {"type": "addhost", "message": "name=value"} to inject an extra_hosts entry into every dependent service — typically its own service name aliased to host-gateway, so consumers keep addressing it by the name they already use while the resource actually lives on the host. Injection relies on plan-node copies sharing the underlying maps, so provider-dependent services get their ExtraHosts materialized before the plan is built. - docs/extension.md documents both, plus the recommended links-style endpoint variables convention (PORT__[_ADDR|_PORT|_PROTO] over setenv) so consumers look endpoints up by the container port they know while providers assign actual host ports freely. The example provider demonstrates the round trip, backed by an e2e scenario; unit tests drive executePlugin against a helper-process provider and cover the injection. Signed-off-by: Nicolas De Loof --- docs/examples/provider.go | 16 ++ docs/extension.md | 47 ++++++ pkg/compose/create.go | 1 + pkg/compose/plugins.go | 85 +++++++++-- pkg/compose/plugins_control_test.go | 142 ++++++++++++++++++ pkg/e2e/providers_test.go | 10 ++ .../TestProviderControlChannel/compose.yaml | 13 ++ 7 files changed, 302 insertions(+), 12 deletions(-) create mode 100644 pkg/compose/plugins_control_test.go create mode 100644 pkg/e2e/testdata/TestProviderControlChannel/compose.yaml diff --git a/docs/examples/provider.go b/docs/examples/provider.go index 8fa5635e12b..be1f08fcd43 100644 --- a/docs/examples/provider.go +++ b/docs/examples/provider.go @@ -91,6 +91,22 @@ func up(options options, args []string) { servicename := args[0] fmt.Printf(`{ "type": "debug", "message": "Starting %s" }%s`, servicename, lineSeparator) + // Ask the running Compose process for the resolved definition of the + // service this provider manages. A Compose that predates the message + // aborts on it, so only providers that require the configuration + // should send it. + fmt.Printf(`{ "type": "get-service-config" }%s`, lineSeparator) + var config struct { + Provider struct { + Type string `json:"type"` + } `json:"provider"` + } + if err := json.NewDecoder(os.Stdin).Decode(&config); err != nil { + fmt.Printf(`{ "type": "error", "message": "get-service-config failed: %v" }%s`, err, lineSeparator) + return + } + fmt.Printf(`{ "type": "setenv", "message": "CONFIG_TYPE=%s" }%s`, config.Provider.Type, lineSeparator) + for i := 0; i < options.size; i += 10 { time.Sleep(1 * time.Second) fmt.Printf(`{ "type": "info", "message": "Processing ... %d%%" }%s`, i*100/options.size, lineSeparator) diff --git a/docs/extension.md b/docs/extension.md index 268f56cc049..f3d281c17a5 100644 --- a/docs/extension.md +++ b/docs/extension.md @@ -59,6 +59,53 @@ JSON messages MUST include a `type` and a `message` attribute. - `setenv`: Lets the plugin tell Compose how dependent services can access the created resource. The variable is automatically prefixed with the service name. See next section for further details. - `rawsetenv`: Same as `setenv`, but the variable is injected as-is without the service name prefix. Useful when applications require exact variable names that cannot be altered. - `debug`: Those messages could help debugging the provider, but are not rendered to the user by default. They are rendered when Compose is started with `--verbose` flag. +- `addhost`: Injects an `extra_hosts` entry into every service depending on the provider service. The message is + `"hostname=value"`, where value is an IP or the special `host-gateway`. A provider that exposes its resource on + the host (published ports) typically sends its own service name — `{"type": "addhost", "message": "database=host-gateway"}` — + so consumers reach it by the service name they already use, e.g. `http://database:5734`. + +### Recommended convention: links-style endpoint variables + +A provider whose resource listens on network ports should describe each endpoint with `setenv` variables following +the legacy [docker links](https://docs.docker.com/engine/network/links/) naming: the variable name is keyed by the +**port the application knows** (the container port), the value carries where that port is actually reachable. With +the automatic service-name prefix, a consumer of a `database` provider managing a resource whose port 5432 is +reachable at `database:31002` (via an `addhost` alias) sees: + +``` +DATABASE_PORT = tcp://database:31002 (primary port: the first one declared) +DATABASE_PORT_5432_TCP = tcp://database:31002 +DATABASE_PORT_5432_TCP_ADDR = database +DATABASE_PORT_5432_TCP_PORT = 31002 +DATABASE_PORT_5432_TCP_PROTO = tcp +``` + +This lets the provider assign actual ports freely (avoiding host port collisions between projects and providers) +while consumers look endpoints up by the well-known port number. +- `get-service-config`: Asks Compose for the resolved configuration of the service the provider manages. See next section. + +## Requesting the service configuration + +A provider can ask the running Compose process for the resolved definition of the service it manages — +the exact model Compose is executing, not a re-resolution. The request is a regular JSON line on `stdout`: +```json +{ "type": "get-service-config" } +``` + +Compose answers on the provider's `stdin` with one JSON line: the resolved, canonical JSON of the service — +the same shape as this service's entry in `docker compose config --format json`, after interpolation and +normalization: +```json +{ "image": "mysql:8", "environment": { "...": "..." } } +``` + +There is no parameter: a provider can only obtain the definition of its own service. The message can be sent +several times; each occurrence is answered with one line. + +Compose versions that predate this message treat it as a protocol error and abort the command, and never +write anything to the provider's `stdin` (the provider reads EOF). A provider that requires the service +configuration should treat EOF as "this Compose version does not support provider requests" and report an +actionable error; a provider that can operate without it should simply not send the message. ```mermaid sequenceDiagram diff --git a/pkg/compose/create.go b/pkg/compose/create.go index b30ce1a65c1..e8932dc8fb8 100644 --- a/pkg/compose/create.go +++ b/pkg/compose/create.go @@ -91,6 +91,7 @@ func (s *composeService) create(ctx context.Context, project *types.Project, opt } prepareNetworks(project) + prepareProviderInjection(project) externalNetworks, err := s.checkExternalNetworks(ctx, project) if err != nil { return err diff --git a/pkg/compose/plugins.go b/pkg/compose/plugins.go index 1514761b47d..b9a41b70ad0 100644 --- a/pkg/compose/plugins.go +++ b/pkg/compose/plugins.go @@ -41,7 +41,7 @@ import ( type JsonMessage struct { Type string `json:"type"` - Message string `json:"message"` + Message string `json:"message,omitempty"` } const ( @@ -50,16 +50,44 @@ const ( SetEnvType = "setenv" RawSetEnvType = "rawsetenv" DebugType = "debug" + AddHostType = "addhost" providerMetadataDirectory = "compose/providers" + + // GetServiceConfigType is a message the provider sends to receive, on + // its stdin, one JSON line holding the resolved canonical configuration + // of the service it manages — answered from the in-memory model. + GetServiceConfigType = "get-service-config" ) type pluginVariables struct { prefixed types.Mapping raw types.Mapping + // hosts are extra_hosts entries ("hostname=value" addhost messages) to + // inject into dependent services, letting them address the provider's + // resource by name — typically "=host-gateway" for a resource + // published on the host. + hosts types.Mapping } var mux sync.Mutex +// prepareProviderInjection makes every provider-dependent service ready to +// receive injections from injectPluginVariables. It must run before the plan +// is built: plan nodes hold value copies of ServiceConfig, so an injection is +// only visible to them through a map that already existed — and was therefore +// shared — when the copy was made. Environment always exists on a loaded +// project; ExtraHosts may be nil and is materialized here. +func prepareProviderInjection(project *types.Project) { + for name, s := range project.Services { + for dep := range s.DependsOn { + if svc, ok := project.Services[dep]; ok && svc.Provider != nil && s.ExtraHosts == nil { + s.ExtraHosts = types.HostsList{} + project.Services[name] = s + } + } + } +} + func (s *composeService) runPlugin(ctx context.Context, project *types.Project, service types.ServiceConfig, command string) error { provider := *service.Provider @@ -85,24 +113,38 @@ func (s *composeService) runPlugin(ctx context.Context, project *types.Project, return nil } + injectPluginVariables(project, service, variables) + return nil +} + +// injectPluginVariables applies what the provider declared to every service +// that depends on it: setenv variables prefixed with the provider service +// name, rawsetenv variables as-is, and addhost entries as extra_hosts. +func injectPluginVariables(project *types.Project, service types.ServiceConfig, variables pluginVariables) { mux.Lock() defer mux.Unlock() for name, s := range project.Services { - if _, ok := s.DependsOn[service.Name]; ok { - prefix := strings.ToUpper(service.Name) + "_" - for key, val := range variables.prefixed { - s.Environment[prefix+key] = &val + if _, ok := s.DependsOn[service.Name]; !ok { + continue + } + prefix := strings.ToUpper(service.Name) + "_" + for key, val := range variables.prefixed { + s.Environment[prefix+key] = &val + } + for key, val := range variables.raw { + if existing, ok := s.Environment[key]; ok && (existing == nil || *existing != val) { + logrus.Warnf("provider %q overrides environment variable %q in service %q", service.Name, key, name) } - for key, val := range variables.raw { - if existing, ok := s.Environment[key]; ok && (existing == nil || *existing != val) { - logrus.Warnf("provider %q overrides environment variable %q in service %q", service.Name, key, name) - } - s.Environment[key] = &val + s.Environment[key] = &val + } + for host, val := range variables.hosts { + if _, ok := s.ExtraHosts[host]; ok { + logrus.Warnf("provider %q overrides extra_hosts entry %q in service %q", service.Name, host, name) } - project.Services[name] = s + s.ExtraHosts[host] = []string{val} } + project.Services[name] = s } - return nil } func (s *composeService) executePlugin(cmd *exec.Cmd, command string, service types.ServiceConfig) (pluginVariables, error) { @@ -125,6 +167,14 @@ func (s *composeService) executePlugin(cmd *exec.Cmd, command string, service ty if err != nil { return pluginVariables{}, err } + stdin, err := cmd.StdinPipe() + if err != nil { + return pluginVariables{}, err + } + // closing stdin on exit unblocks a provider waiting for a response the + // loop will never produce (e.g. a request emitted after an error) + defer func() { _ = stdin.Close() }() + responses := json.NewEncoder(stdin) err = cmd.Start() if err != nil { @@ -137,6 +187,7 @@ func (s *composeService) executePlugin(cmd *exec.Cmd, command string, service ty variables := pluginVariables{ prefixed: types.Mapping{}, raw: types.Mapping{}, + hosts: types.Mapping{}, } for { @@ -166,6 +217,16 @@ func (s *composeService) executePlugin(cmd *exec.Cmd, command string, service ty return pluginVariables{}, fmt.Errorf("invalid response from plugin: %s", msg.Message) } variables.raw[key] = val + case AddHostType: + key, val, found := strings.Cut(msg.Message, "=") + if !found { + return pluginVariables{}, fmt.Errorf("invalid response from plugin: %s", msg.Message) + } + variables.hosts[key] = val + case GetServiceConfigType: + if err := responses.Encode(service); err != nil { + return pluginVariables{}, fmt.Errorf("failed to answer get-service-config: %w", err) + } case DebugType: logrus.Debugf("%s: %s", service.Name, msg.Message) default: diff --git a/pkg/compose/plugins_control_test.go b/pkg/compose/plugins_control_test.go new file mode 100644 index 00000000000..a7beed5207c --- /dev/null +++ b/pkg/compose/plugins_control_test.go @@ -0,0 +1,142 @@ +/* + Copyright 2020 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package compose + +import ( + "bufio" + "encoding/json" + "fmt" + "os" + "os/exec" + "testing" + + "github.com/compose-spec/compose-go/v2/types" + "go.uber.org/mock/gomock" + "gotest.tools/v3/assert" + + "github.com/docker/compose/v5/pkg/mocks" +) + +// TestExecutePlugin_GetServiceConfig runs executePlugin against a fake +// provider (this test binary re-executed, see TestHelperProviderConfig): each +// get-service-config message must be answered on the provider's stdin with +// one JSON line holding the in-memory service's canonical configuration. +func TestExecutePlugin_GetServiceConfig(t *testing.T) { + mockCtrl := gomock.NewController(t) + cli := mocks.NewMockCli(mockCtrl) + cli.EXPECT().Client().Return(mocks.NewMockAPIClient(mockCtrl)).AnyTimes() + svc, err := NewComposeService(cli, WithEventProcessor(noopEventProcessor{})) + assert.NilError(t, err) + + cmd := exec.Command(os.Args[0], "-test.run=TestHelperProviderConfig") + cmd.Env = append(os.Environ(), "GO_WANT_HELPER_PROCESS=1") + + service := types.ServiceConfig{ + Name: "db", + Provider: &types.ServiceProviderConfig{ + Type: "sbx", + Options: types.MultiOptions{"template": {"agent:latest"}}, + }, + } + variables, err := svc.(*composeService).executePlugin(cmd, "up", service) + assert.NilError(t, err) + assert.Equal(t, variables.prefixed["TEMPLATE"], "agent:latest") + // the channel stays usable for more than one request + assert.Equal(t, variables.prefixed["TEMPLATE_AGAIN"], "agent:latest") + assert.Equal(t, variables.hosts["db"], "host-gateway") +} + +// addhost entries reach dependent services as extra_hosts; setenv variables +// as prefixed environment. +func TestInjectPluginVariables(t *testing.T) { + db := types.ServiceConfig{Name: "db", Provider: &types.ServiceProviderConfig{Type: "sbx"}} + project := &types.Project{ + Name: "test", + Services: types.Services{ + "db": db, + "app": { + Name: "app", + DependsOn: types.DependsOnConfig{"db": {}}, + Environment: types.MappingWithEquals{}, + }, + "other": { + Name: "other", + Environment: types.MappingWithEquals{}, + }, + }, + } + // the create/up path materializes the maps injection mutates before the + // plan copies services; the injection relies on that sharing + prepareProviderInjection(project) + injectPluginVariables(project, db, pluginVariables{ + prefixed: types.Mapping{"PORT": "5734"}, + raw: types.Mapping{}, + hosts: types.Mapping{"db": "host-gateway"}, + }) + + app := project.Services["app"] + assert.Equal(t, *app.Environment["DB_PORT"], "5734") + assert.DeepEqual(t, app.ExtraHosts["db"], []string{"host-gateway"}) + + other := project.Services["other"] + assert.Assert(t, other.ExtraHosts == nil) + _, injected := other.Environment["DB_PORT"] + assert.Assert(t, !injected) +} + +// TestHelperProviderConfig is not a test: it is the fake provider process +// spawned by TestExecutePlugin_GetServiceConfig. +func TestHelperProviderConfig(t *testing.T) { + if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" { + t.Skip("helper process for TestExecutePlugin_GetServiceConfig") + } + stdin := bufio.NewReader(os.Stdin) + emit := func(msg JsonMessage) { + if err := json.NewEncoder(os.Stdout).Encode(msg); err != nil { + os.Exit(1) + } + } + getServiceConfig := func() (template string, ok bool) { + emit(JsonMessage{Type: GetServiceConfigType}) + line, err := stdin.ReadBytes('\n') + if err != nil { + emit(JsonMessage{Type: ErrorType, Message: fmt.Sprintf("reading service config: %v", err)}) + return "", false + } + var config struct { + Provider struct { + Options map[string][]string `json:"options"` + } `json:"provider"` + } + if err := json.Unmarshal(line, &config); err != nil { + emit(JsonMessage{Type: ErrorType, Message: fmt.Sprintf("bad service config %s: %v", line, err)}) + return "", false + } + return config.Provider.Options["template"][0], true + } + + template, ok := getServiceConfig() + if !ok { + os.Exit(0) + } + emit(JsonMessage{Type: SetEnvType, Message: "TEMPLATE=" + template}) + emit(JsonMessage{Type: AddHostType, Message: "db=host-gateway"}) + if template, ok = getServiceConfig(); ok { + emit(JsonMessage{Type: SetEnvType, Message: "TEMPLATE_AGAIN=" + template}) + } + os.Exit(0) +} diff --git a/pkg/e2e/providers_test.go b/pkg/e2e/providers_test.go index 0b8031c3f4a..c2b5245f3ec 100644 --- a/pkg/e2e/providers_test.go +++ b/pkg/e2e/providers_test.go @@ -61,6 +61,16 @@ func TestDependsOnMultipleProviders(t *testing.T) { OutputContains("test-1 | PROVIDER2_URL=https://magic.cloud/provider2")) } +func TestProviderControlChannel(t *testing.T) { + // The example provider requests its own resolved service config over the + // stdio control channel and reflects provider.type back through setenv: + // the dependent service seeing DB_CONFIG_TYPE proves the round trip. + providerScenario(t, "a provider must be able to request its resolved service config from the running compose process"). + Step("the dependent service sees the value the provider read from its config", + ComposeCmd("up"), + OutputContains("test-1 | DB_CONFIG_TYPE=example-provider")) +} + func TestProviderRawSetEnv(t *testing.T) { providerScenario(t, "setenv variables must be service-prefixed, rawsetenv injected as-is"). Step("the service sees both variable flavors", diff --git a/pkg/e2e/testdata/TestProviderControlChannel/compose.yaml b/pkg/e2e/testdata/TestProviderControlChannel/compose.yaml new file mode 100644 index 00000000000..5c5ff221e49 --- /dev/null +++ b/pkg/e2e/testdata/TestProviderControlChannel/compose.yaml @@ -0,0 +1,13 @@ +services: + test: + image: alpine + command: env + depends_on: + - db + db: + provider: + type: example-provider + options: + name: db + type: test1 + size: 1