feat(provider): get-service-config, addhost, and endpoint conventions

docs/examples/provider.go

@@ -91,6 +91,22 @@ func up(options options, args []string) {
 	servicename := args[0]
 	fmt.Printf(`{ "type": "debug", "message": "Starting %s" }%s`, servicename, lineSeparator)
 
+	// Ask the running Compose process for the resolved definition of the
+	// service this provider manages. A Compose that predates the message
+	// aborts on it, so only providers that require the configuration
+	// should send it.
+	fmt.Printf(`{ "type": "get-service-config" }%s`, lineSeparator)
+	var config struct {
+		Provider struct {
+			Type string `json:"type"`
+		} `json:"provider"`
+	}
+	if err := json.NewDecoder(os.Stdin).Decode(&config); err != nil {
+		fmt.Printf(`{ "type": "error", "message": "get-service-config failed: %v" }%s`, err, lineSeparator)
+		return
+	}
+	fmt.Printf(`{ "type": "setenv", "message": "CONFIG_TYPE=%s" }%s`, config.Provider.Type, lineSeparator)
+
 	for i := 0; i < options.size; i += 10 {
 		time.Sleep(1 * time.Second)
 		fmt.Printf(`{ "type": "info", "message": "Processing ... %d%%" }%s`, i*100/options.size, lineSeparator)

docs/extension.md

@@ -59,6 +59,53 @@ JSON messages MUST include a `type` and a `message` attribute.
 - `setenv`: Lets the plugin tell Compose how dependent services can access the created resource. The variable is automatically prefixed with the service name. See next section for further details.
 - `rawsetenv`: Same as `setenv`, but the variable is injected as-is without the service name prefix. Useful when applications require exact variable names that cannot be altered.
 - `debug`: Those messages could help debugging the provider, but are not rendered to the user by default. They are rendered when Compose is started with `--verbose` flag.
+- `addhost`: Injects an `extra_hosts` entry into every service depending on the provider service. The message is
+  `"hostname=value"`, where value is an IP or the special `host-gateway`. A provider that exposes its resource on
+  the host (published ports) typically sends its own service name — `{"type": "addhost", "message": "database=host-gateway"}` —
+  so consumers reach it by the service name they already use, e.g. `http://database:5734`.
+
+### Recommended convention: links-style endpoint variables
+
+A provider whose resource listens on network ports should describe each endpoint with `setenv` variables following
+the legacy [docker links](https://docs.docker.com/engine/network/links/) naming: the variable name is keyed by the
+**port the application knows** (the container port), the value carries where that port is actually reachable. With
+the automatic service-name prefix, a consumer of a `database` provider managing a resource whose port 5432 is
+reachable at `database:31002` (via an `addhost` alias) sees:
+
+```
+DATABASE_PORT                 = tcp://database:31002    (primary port: the first one declared)
+DATABASE_PORT_5432_TCP        = tcp://database:31002
+DATABASE_PORT_5432_TCP_ADDR   = database
+DATABASE_PORT_5432_TCP_PORT   = 31002
+DATABASE_PORT_5432_TCP_PROTO  = tcp
+```
+
+This lets the provider assign actual ports freely (avoiding host port collisions between projects and providers)
+while consumers look endpoints up by the well-known port number.
+- `get-service-config`: Asks Compose for the resolved configuration of the service the provider manages. See next section.
+
+## Requesting the service configuration
+
+A provider can ask the running Compose process for the resolved definition of the service it manages —
+the exact model Compose is executing, not a re-resolution. The request is a regular JSON line on `stdout`:
+```json
+{ "type": "get-service-config" }
+```
+
+Compose answers on the provider's `stdin` with one JSON line: the resolved, canonical JSON of the service —
+the same shape as this service's entry in `docker compose config --format json`, after interpolation and
+normalization:
+```json
+{ "image": "mysql:8", "environment": { "...": "..." } }
+```
+
+There is no parameter: a provider can only obtain the definition of its own service. The message can be sent
+several times; each occurrence is answered with one line.
+
+Compose versions that predate this message treat it as a protocol error and abort the command, and never
+write anything to the provider's `stdin` (the provider reads EOF). A provider that requires the service
+configuration should treat EOF as "this Compose version does not support provider requests" and report an
+actionable error; a provider that can operate without it should simply not send the message.
 
 ```mermaid
 sequenceDiagram

pkg/compose/create.go

@@ -91,6 +91,7 @@ func (s *composeService) create(ctx context.Context, project *types.Project, opt
 	}
 
 	prepareNetworks(project)
+	prepareProviderInjection(project)
 	externalNetworks, err := s.checkExternalNetworks(ctx, project)
 	if err != nil {
 		return err

pkg/compose/plugins.go

@@ -41,7 +41,7 @@ import (
 
 type JsonMessage struct {
 	Type    string `json:"type"`
-	Message string `json:"message"`
+	Message string `json:"message,omitempty"`
 }
 
 const (
@@ -50,16 +50,44 @@ const (
 	SetEnvType                = "setenv"
 	RawSetEnvType             = "rawsetenv"
 	DebugType                 = "debug"
+	AddHostType               = "addhost"
 	providerMetadataDirectory = "compose/providers"
+
+	// GetServiceConfigType is a message the provider sends to receive, on
+	// its stdin, one JSON line holding the resolved canonical configuration
+	// of the service it manages — answered from the in-memory model.
+	GetServiceConfigType = "get-service-config"
 )
 
 type pluginVariables struct {
 	prefixed types.Mapping
 	raw      types.Mapping
+	// hosts are extra_hosts entries ("hostname=value" addhost messages) to
+	// inject into dependent services, letting them address the provider's
+	// resource by name — typically "<service>=host-gateway" for a resource
+	// published on the host.
+	hosts types.Mapping
 }
 
 var mux sync.Mutex
 
+// prepareProviderInjection makes every provider-dependent service ready to
+// receive injections from injectPluginVariables. It must run before the plan
+// is built: plan nodes hold value copies of ServiceConfig, so an injection is
+// only visible to them through a map that already existed — and was therefore
+// shared — when the copy was made. Environment always exists on a loaded
+// project; ExtraHosts may be nil and is materialized here.
+func prepareProviderInjection(project *types.Project) {
+	for name, s := range project.Services {
+		for dep := range s.DependsOn {
+			if svc, ok := project.Services[dep]; ok && svc.Provider != nil && s.ExtraHosts == nil {
+				s.ExtraHosts = types.HostsList{}
+				project.Services[name] = s
+			}
+		}
+	}
+}
+
 func (s *composeService) runPlugin(ctx context.Context, project *types.Project, service types.ServiceConfig, command string) error {
 	provider := *service.Provider
 
@@ -85,24 +113,38 @@ func (s *composeService) runPlugin(ctx context.Context, project *types.Project,
 		return nil
 	}
 
+	injectPluginVariables(project, service, variables)
+	return nil
+}
+
+// injectPluginVariables applies what the provider declared to every service
+// that depends on it: setenv variables prefixed with the provider service
+// name, rawsetenv variables as-is, and addhost entries as extra_hosts.
+func injectPluginVariables(project *types.Project, service types.ServiceConfig, variables pluginVariables) {
 	mux.Lock()
 	defer mux.Unlock()
 	for name, s := range project.Services {
-		if _, ok := s.DependsOn[service.Name]; ok {
-			prefix := strings.ToUpper(service.Name) + "_"
-			for key, val := range variables.prefixed {
-				s.Environment[prefix+key] = &val
+		if _, ok := s.DependsOn[service.Name]; !ok {
+			continue
+		}
+		prefix := strings.ToUpper(service.Name) + "_"
+		for key, val := range variables.prefixed {
+			s.Environment[prefix+key] = &val
+		}
+		for key, val := range variables.raw {
+			if existing, ok := s.Environment[key]; ok && (existing == nil || *existing != val) {
+				logrus.Warnf("provider %q overrides environment variable %q in service %q", service.Name, key, name)
 			}
-			for key, val := range variables.raw {
-				if existing, ok := s.Environment[key]; ok && (existing == nil || *existing != val) {
-					logrus.Warnf("provider %q overrides environment variable %q in service %q", service.Name, key, name)
-				}
-				s.Environment[key] = &val
+			s.Environment[key] = &val
+		}
+		for host, val := range variables.hosts {
+			if _, ok := s.ExtraHosts[host]; ok {
+				logrus.Warnf("provider %q overrides extra_hosts entry %q in service %q", service.Name, host, name)
 			}
-			project.Services[name] = s
+			s.ExtraHosts[host] = []string{val}
 		}
+		project.Services[name] = s
 	}
-	return nil
 }
 
 func (s *composeService) executePlugin(cmd *exec.Cmd, command string, service types.ServiceConfig) (pluginVariables, error) {
@@ -125,6 +167,14 @@ func (s *composeService) executePlugin(cmd *exec.Cmd, command string, service ty
 	if err != nil {
 		return pluginVariables{}, err
 	}
+	stdin, err := cmd.StdinPipe()
+	if err != nil {
+		return pluginVariables{}, err
+	}
+	// closing stdin on exit unblocks a provider waiting for a response the
+	// loop will never produce (e.g. a request emitted after an error)
+	defer func() { _ = stdin.Close() }()
+	responses := json.NewEncoder(stdin)
 
 	err = cmd.Start()
 	if err != nil {
@@ -137,6 +187,7 @@ func (s *composeService) executePlugin(cmd *exec.Cmd, command string, service ty
 	variables := pluginVariables{
 		prefixed: types.Mapping{},
 		raw:      types.Mapping{},
+		hosts:    types.Mapping{},
 	}
 
 	for {
@@ -166,6 +217,16 @@ func (s *composeService) executePlugin(cmd *exec.Cmd, command string, service ty
 				return pluginVariables{}, fmt.Errorf("invalid response from plugin: %s", msg.Message)
 			}
 			variables.raw[key] = val
+		case AddHostType:
+			key, val, found := strings.Cut(msg.Message, "=")
+			if !found {
+				return pluginVariables{}, fmt.Errorf("invalid response from plugin: %s", msg.Message)
+			}
+			variables.hosts[key] = val
+		case GetServiceConfigType:
+			if err := responses.Encode(service); err != nil {
+				return pluginVariables{}, fmt.Errorf("failed to answer get-service-config: %w", err)
+			}
 		case DebugType:
 			logrus.Debugf("%s: %s", service.Name, msg.Message)
 		default:

pkg/compose/plugins_control_test.go

@@ -0,0 +1,142 @@
+/*
+   Copyright 2020 Docker Compose CLI authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package compose
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"os"
+	"os/exec"
+	"testing"
+
+	"github.com/compose-spec/compose-go/v2/types"
+	"go.uber.org/mock/gomock"
+	"gotest.tools/v3/assert"
+
+	"github.com/docker/compose/v5/pkg/mocks"
+)
+
+// TestExecutePlugin_GetServiceConfig runs executePlugin against a fake
+// provider (this test binary re-executed, see TestHelperProviderConfig): each
+// get-service-config message must be answered on the provider's stdin with
+// one JSON line holding the in-memory service's canonical configuration.
+func TestExecutePlugin_GetServiceConfig(t *testing.T) {
+	mockCtrl := gomock.NewController(t)
+	cli := mocks.NewMockCli(mockCtrl)
+	cli.EXPECT().Client().Return(mocks.NewMockAPIClient(mockCtrl)).AnyTimes()
+	svc, err := NewComposeService(cli, WithEventProcessor(noopEventProcessor{}))
+	assert.NilError(t, err)
+
+	cmd := exec.Command(os.Args[0], "-test.run=TestHelperProviderConfig")
+	cmd.Env = append(os.Environ(), "GO_WANT_HELPER_PROCESS=1")
+
+	service := types.ServiceConfig{
+		Name: "db",
+		Provider: &types.ServiceProviderConfig{
+			Type:    "sbx",
+			Options: types.MultiOptions{"template": {"agent:latest"}},
+		},
+	}
+	variables, err := svc.(*composeService).executePlugin(cmd, "up", service)
+	assert.NilError(t, err)
+	assert.Equal(t, variables.prefixed["TEMPLATE"], "agent:latest")
+	// the channel stays usable for more than one request
+	assert.Equal(t, variables.prefixed["TEMPLATE_AGAIN"], "agent:latest")
+	assert.Equal(t, variables.hosts["db"], "host-gateway")
+}
+
+// addhost entries reach dependent services as extra_hosts; setenv variables
+// as prefixed environment.
+func TestInjectPluginVariables(t *testing.T) {
+	db := types.ServiceConfig{Name: "db", Provider: &types.ServiceProviderConfig{Type: "sbx"}}
+	project := &types.Project{
+		Name: "test",
+		Services: types.Services{
+			"db": db,
+			"app": {
+				Name:        "app",
+				DependsOn:   types.DependsOnConfig{"db": {}},
+				Environment: types.MappingWithEquals{},
+			},
+			"other": {
+				Name:        "other",
+				Environment: types.MappingWithEquals{},
+			},
+		},
+	}
+	// the create/up path materializes the maps injection mutates before the
+	// plan copies services; the injection relies on that sharing
+	prepareProviderInjection(project)
+	injectPluginVariables(project, db, pluginVariables{
+		prefixed: types.Mapping{"PORT": "5734"},
+		raw:      types.Mapping{},
+		hosts:    types.Mapping{"db": "host-gateway"},
+	})
+
+	app := project.Services["app"]
+	assert.Equal(t, *app.Environment["DB_PORT"], "5734")
+	assert.DeepEqual(t, app.ExtraHosts["db"], []string{"host-gateway"})
+
+	other := project.Services["other"]
+	assert.Assert(t, other.ExtraHosts == nil)
+	_, injected := other.Environment["DB_PORT"]
+	assert.Assert(t, !injected)
+}
+
+// TestHelperProviderConfig is not a test: it is the fake provider process
+// spawned by TestExecutePlugin_GetServiceConfig.
+func TestHelperProviderConfig(t *testing.T) {
+	if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" {
+		t.Skip("helper process for TestExecutePlugin_GetServiceConfig")
+	}
+	stdin := bufio.NewReader(os.Stdin)
+	emit := func(msg JsonMessage) {
+		if err := json.NewEncoder(os.Stdout).Encode(msg); err != nil {
+			os.Exit(1)
+		}
+	}
+	getServiceConfig := func() (template string, ok bool) {
+		emit(JsonMessage{Type: GetServiceConfigType})
+		line, err := stdin.ReadBytes('\n')
+		if err != nil {
+			emit(JsonMessage{Type: ErrorType, Message: fmt.Sprintf("reading service config: %v", err)})
+			return "", false
+		}
+		var config struct {
+			Provider struct {
+				Options map[string][]string `json:"options"`
+			} `json:"provider"`
+		}
+		if err := json.Unmarshal(line, &config); err != nil {
+			emit(JsonMessage{Type: ErrorType, Message: fmt.Sprintf("bad service config %s: %v", line, err)})
+			return "", false
+		}
+		return config.Provider.Options["template"][0], true
+	}
+
+	template, ok := getServiceConfig()
+	if !ok {
+		os.Exit(0)
+	}
+	emit(JsonMessage{Type: SetEnvType, Message: "TEMPLATE=" + template})
+	emit(JsonMessage{Type: AddHostType, Message: "db=host-gateway"})
+	if template, ok = getServiceConfig(); ok {
+		emit(JsonMessage{Type: SetEnvType, Message: "TEMPLATE_AGAIN=" + template})
+	}
+	os.Exit(0)
+}

pkg/e2e/providers_test.go

@@ -61,6 +61,16 @@ func TestDependsOnMultipleProviders(t *testing.T) {
 			OutputContains("test-1  | PROVIDER2_URL=https://magic.cloud/provider2"))
 }
 
+func TestProviderControlChannel(t *testing.T) {
+	// The example provider requests its own resolved service config over the
+	// stdio control channel and reflects provider.type back through setenv:
+	// the dependent service seeing DB_CONFIG_TYPE proves the round trip.
+	providerScenario(t, "a provider must be able to request its resolved service config from the running compose process").
+		Step("the dependent service sees the value the provider read from its config",
+			ComposeCmd("up"),
+			OutputContains("test-1  | DB_CONFIG_TYPE=example-provider"))
+}
+
 func TestProviderRawSetEnv(t *testing.T) {
 	providerScenario(t, "setenv variables must be service-prefixed, rawsetenv injected as-is").
 		Step("the service sees both variable flavors",