diff --git a/docs/examples/provider.go b/docs/examples/provider.go index 8fa5635e12..83ab901993 100644 --- a/docs/examples/provider.go +++ b/docs/examples/provider.go @@ -91,6 +91,30 @@ func up(options options, args []string) { servicename := args[0] fmt.Printf(`{ "type": "debug", "message": "Starting %s" }%s`, servicename, lineSeparator) + // Ask the running Compose process for the resolved definition of the + // service this provider manages. A Compose that predates the message + // aborts on it, so only providers that require the configuration + // should send it. The decoder must be created once and reused across + // requests: it reads ahead, so a fresh decoder per request would discard + // buffered bytes and hang on the next answer. + responses := json.NewDecoder(os.Stdin) + fmt.Printf(`{ "type": "get-service-config" }%s`, lineSeparator) + var config struct { + Provider struct { + Type string `json:"type"` + } `json:"provider"` + } + if err := responses.Decode(&config); err != nil { + // error text is not JSON-safe either: encode, don't interpolate + msg, _ := json.Marshal(map[string]string{"type": "error", "message": fmt.Sprintf("get-service-config failed: %v", err)}) + fmt.Println(string(msg)) + return + } + // values read from the configuration are not necessarily JSON-safe: + // encode the message instead of interpolating it into a JSON literal + setenv, _ := json.Marshal(map[string]string{"type": "setenv", "message": "CONFIG_TYPE=" + config.Provider.Type}) + fmt.Println(string(setenv)) + for i := 0; i < options.size; i += 10 { time.Sleep(1 * time.Second) fmt.Printf(`{ "type": "info", "message": "Processing ... %d%%" }%s`, i*100/options.size, lineSeparator) diff --git a/docs/extension.md b/docs/extension.md index 268f56cc04..4a06f8cb01 100644 --- a/docs/extension.md +++ b/docs/extension.md @@ -59,6 +59,30 @@ JSON messages MUST include a `type` and a `message` attribute. - `setenv`: Lets the plugin tell Compose how dependent services can access the created resource. The variable is automatically prefixed with the service name. See next section for further details. - `rawsetenv`: Same as `setenv`, but the variable is injected as-is without the service name prefix. Useful when applications require exact variable names that cannot be altered. - `debug`: Those messages could help debugging the provider, but are not rendered to the user by default. They are rendered when Compose is started with `--verbose` flag. +- `get-service-config`: Asks Compose for the resolved configuration of the service the provider manages. See next section. + +## Requesting the service configuration + +A provider can ask the running Compose process for the resolved definition of the service it manages — +the exact model Compose is executing, not a re-resolution. The request is a regular JSON line on `stdout`: +```json +{ "type": "get-service-config" } +``` + +Compose answers on the provider's `stdin` with one JSON line: the resolved, canonical JSON of the service — +the same shape as this service's entry in `docker compose config --format json`, after interpolation and +normalization: +```json +{ "image": "mysql:8", "environment": { "...": "..." } } +``` + +There is no parameter: a provider can only obtain the definition of its own service. The message can be sent +several times; each occurrence is answered with one line. + +Compose versions that predate this message treat it as a protocol error and abort the command, and never +write anything to the provider's `stdin` (the provider reads EOF). A provider that requires the service +configuration should treat EOF as "this Compose version does not support provider requests" and report an +actionable error; a provider that can operate without it should simply not send the message. ```mermaid sequenceDiagram diff --git a/pkg/compose/plugins.go b/pkg/compose/plugins.go index 1514761b47..84a40775e0 100644 --- a/pkg/compose/plugins.go +++ b/pkg/compose/plugins.go @@ -41,7 +41,7 @@ import ( type JsonMessage struct { Type string `json:"type"` - Message string `json:"message"` + Message string `json:"message,omitempty"` } const ( @@ -51,6 +51,11 @@ const ( RawSetEnvType = "rawsetenv" DebugType = "debug" providerMetadataDirectory = "compose/providers" + + // GetServiceConfigType is a message the provider sends to receive, on + // its stdin, one JSON line holding the resolved canonical configuration + // of the service it manages — answered from the in-memory model. + GetServiceConfigType = "get-service-config" ) type pluginVariables struct { @@ -125,11 +130,51 @@ func (s *composeService) executePlugin(cmd *exec.Cmd, command string, service ty if err != nil { return pluginVariables{}, err } + stdin, err := cmd.StdinPipe() + if err != nil { + return pluginVariables{}, err + } + // Answers are written from their own goroutine: writing a config larger + // than the OS pipe buffer from the read loop would deadlock against a + // provider that emits stdout before draining its stdin. Every answer is + // the same serialized service, so completion order is irrelevant; the + // mutex keeps individual writes atomic. Write failures are not reported + // from here — a provider missing its answer reads EOF once stdin closes. + var stdinMu sync.Mutex + var answers sync.WaitGroup + processExited := false + // Closing stdin on exit unblocks a provider waiting for a response the + // loop will never produce (e.g. a request emitted after an error). An + // answer dispatched but not yet written must land before the close — + // but only once the process has exited can a write not block forever + // (a dead peer turns it into EPIPE); on error paths the provider may + // still be alive and not reading, so close first to error the write + // out instead of hanging the wait. + defer func() { + if processExited { + answers.Wait() + _ = stdin.Close() + } else { + _ = stdin.Close() + answers.Wait() + } + }() err = cmd.Start() if err != nil { return pluginVariables{}, err } + // Error paths return before the normal cmd.Wait below and would leave + // the provider as a zombie (and possibly running): reap it — kill + // first, as it may be misbehaving or blocked, which also errors out + // any in-flight answer write. Runs before the stdin/answers defer + // above (LIFO). + defer func() { + if !processExited { + _ = cmd.Process.Kill() + _ = cmd.Wait() + } + }() decoder := json.NewDecoder(stdout) defer func() { _ = stdout.Close() }() @@ -166,6 +211,19 @@ func (s *composeService) executePlugin(cmd *exec.Cmd, command string, service ty return pluginVariables{}, fmt.Errorf("invalid response from plugin: %s", msg.Message) } variables.raw[key] = val + case GetServiceConfigType: + payload, err := json.Marshal(service) + if err != nil { + return pluginVariables{}, fmt.Errorf("failed to answer get-service-config: %w", err) + } + payload = append(payload, '\n') + answers.Add(1) + go func() { + defer answers.Done() + stdinMu.Lock() + defer stdinMu.Unlock() + _, _ = stdin.Write(payload) + }() case DebugType: logrus.Debugf("%s: %s", service.Name, msg.Message) default: @@ -174,6 +232,7 @@ func (s *composeService) executePlugin(cmd *exec.Cmd, command string, service ty } err = cmd.Wait() + processExited = true if err != nil { s.events.On(errorEvent(service.Name, err.Error())) return pluginVariables{}, fmt.Errorf("failed to %s service provider: %s", action, err.Error()) diff --git a/pkg/compose/plugins_control_test.go b/pkg/compose/plugins_control_test.go new file mode 100644 index 0000000000..5d1179c852 --- /dev/null +++ b/pkg/compose/plugins_control_test.go @@ -0,0 +1,96 @@ +/* + Copyright 2020 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package compose + +import ( + "encoding/json" + "fmt" + "os" + "os/exec" + "testing" + + "github.com/compose-spec/compose-go/v2/types" + "go.uber.org/mock/gomock" + "gotest.tools/v3/assert" + + "github.com/docker/compose/v5/pkg/mocks" +) + +// TestExecutePlugin_GetServiceConfig runs executePlugin against a fake +// provider (this test binary re-executed, see TestHelperProviderConfig): each +// get-service-config message must be answered on the provider's stdin with +// one JSON line holding the in-memory service's canonical configuration. +func TestExecutePlugin_GetServiceConfig(t *testing.T) { + mockCtrl := gomock.NewController(t) + cli := mocks.NewMockCli(mockCtrl) + cli.EXPECT().Client().Return(mocks.NewMockAPIClient(mockCtrl)).AnyTimes() + svc, err := NewComposeService(cli, WithEventProcessor(noopEventProcessor{})) + assert.NilError(t, err) + + cmd := exec.Command(os.Args[0], "-test.run=TestHelperProviderConfig") + cmd.Env = append(os.Environ(), "GO_WANT_HELPER_PROCESS=1") + + service := types.ServiceConfig{ + Name: "db", + Provider: &types.ServiceProviderConfig{ + Type: "sbx", + Options: types.MultiOptions{"template": {"agent:latest"}}, + }, + } + variables, err := svc.(*composeService).executePlugin(cmd, "up", service) + assert.NilError(t, err) + assert.Equal(t, variables.prefixed["TEMPLATE"], "agent:latest") + // the channel stays usable for more than one request + assert.Equal(t, variables.prefixed["TEMPLATE_AGAIN"], "agent:latest") +} + +// TestHelperProviderConfig is not a test: it is the fake provider process +// spawned by TestExecutePlugin_GetServiceConfig. +func TestHelperProviderConfig(t *testing.T) { + if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" { + t.Skip("helper process for TestExecutePlugin_GetServiceConfig") + } + responses := json.NewDecoder(os.Stdin) + emit := func(msg JsonMessage) { + if err := json.NewEncoder(os.Stdout).Encode(msg); err != nil { + os.Exit(1) + } + } + getServiceConfig := func() (template string, ok bool) { + emit(JsonMessage{Type: GetServiceConfigType}) + var config struct { + Provider struct { + Options map[string][]string `json:"options"` + } `json:"provider"` + } + if err := responses.Decode(&config); err != nil { + emit(JsonMessage{Type: ErrorType, Message: fmt.Sprintf("reading service config: %v", err)}) + return "", false + } + return config.Provider.Options["template"][0], true + } + + template, ok := getServiceConfig() + if !ok { + os.Exit(0) + } + emit(JsonMessage{Type: SetEnvType, Message: "TEMPLATE=" + template}) + if template, ok = getServiceConfig(); ok { + emit(JsonMessage{Type: SetEnvType, Message: "TEMPLATE_AGAIN=" + template}) + } + os.Exit(0) +} diff --git a/pkg/e2e/providers_test.go b/pkg/e2e/providers_test.go index 0b8031c3f4..c2b5245f3e 100644 --- a/pkg/e2e/providers_test.go +++ b/pkg/e2e/providers_test.go @@ -61,6 +61,16 @@ func TestDependsOnMultipleProviders(t *testing.T) { OutputContains("test-1 | PROVIDER2_URL=https://magic.cloud/provider2")) } +func TestProviderControlChannel(t *testing.T) { + // The example provider requests its own resolved service config over the + // stdio control channel and reflects provider.type back through setenv: + // the dependent service seeing DB_CONFIG_TYPE proves the round trip. + providerScenario(t, "a provider must be able to request its resolved service config from the running compose process"). + Step("the dependent service sees the value the provider read from its config", + ComposeCmd("up"), + OutputContains("test-1 | DB_CONFIG_TYPE=example-provider")) +} + func TestProviderRawSetEnv(t *testing.T) { providerScenario(t, "setenv variables must be service-prefixed, rawsetenv injected as-is"). Step("the service sees both variable flavors", diff --git a/pkg/e2e/testdata/TestProviderControlChannel/compose.yaml b/pkg/e2e/testdata/TestProviderControlChannel/compose.yaml new file mode 100644 index 0000000000..5c5ff221e4 --- /dev/null +++ b/pkg/e2e/testdata/TestProviderControlChannel/compose.yaml @@ -0,0 +1,13 @@ +services: + test: + image: alpine + command: env + depends_on: + - db + db: + provider: + type: example-provider + options: + name: db + type: test1 + size: 1