diff --git a/Directory.Packages.props b/Directory.Packages.props index 5d5161c..459579f 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -15,16 +15,18 @@ - + + - - - - - - + + + + + + + diff --git a/README.md b/README.md index 13ccd52..e2f8e61 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ Prerequisites: * [AspNet](src/AspNet) - Demonstration of a generic host worker and an ASP.NET workflow starter. * [Bedrock](src/Bedrock) - Orchestrate a chatbot with Amazon Bedrock. * [ClientMtls](src/ClientMtls) - How to use client certificate authentication, e.g. for Temporal Cloud. +* [Gcp/CloudRun/OpenTelemetry](src/Gcp/CloudRun/OpenTelemetry) - Run a continuously-polling worker in a Google Cloud Run worker pool, exporting OpenTelemetry metrics and traces to a collector sidecar. * [ContextPropagation](src/ContextPropagation) - Context propagation via interceptors. * [CounterInterceptor](src/CounterInterceptor/) - Simple Workflow and Client Interceptors example. * [DependencyInjection](src/DependencyInjection) - How to inject dependencies in activities and use generic hosts for workers diff --git a/TemporalioSamples.sln b/TemporalioSamples.sln index a3ede5c..5ee8513 100644 --- a/TemporalioSamples.sln +++ b/TemporalioSamples.sln @@ -7,6 +7,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{1A647B41-53D EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TemporalioSamples.ActivityWorker", "src\ActivityWorker\TemporalioSamples.ActivityWorker.csproj", "{7AECC7C6-9A21-4B8A-84D9-AFC4F5840CAF}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TemporalioSamples.Gcp.CloudRun.OpenTelemetry", "src\Gcp\CloudRun\OpenTelemetry\TemporalioSamples.Gcp.CloudRun.OpenTelemetry.csproj", "{E0F934F9-10A7-41A0-A85E-EE6D0E267367}" +EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TemporalioSamples.Tests", "tests\TemporalioSamples.Tests.csproj", "{3FA7E5DF-03B7-4586-A980-85C155B376C5}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "AspNet", "AspNet", "{E431D279-E02B-4670-B934-3DB9F15D8CCC}" @@ -163,6 +165,18 @@ Global {7AECC7C6-9A21-4B8A-84D9-AFC4F5840CAF}.Release|x64.Build.0 = Release|Any CPU {7AECC7C6-9A21-4B8A-84D9-AFC4F5840CAF}.Release|x86.ActiveCfg = Release|Any CPU {7AECC7C6-9A21-4B8A-84D9-AFC4F5840CAF}.Release|x86.Build.0 = Release|Any CPU + {E0F934F9-10A7-41A0-A85E-EE6D0E267367}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E0F934F9-10A7-41A0-A85E-EE6D0E267367}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E0F934F9-10A7-41A0-A85E-EE6D0E267367}.Debug|x64.ActiveCfg = Debug|Any CPU + {E0F934F9-10A7-41A0-A85E-EE6D0E267367}.Debug|x64.Build.0 = Debug|Any CPU + {E0F934F9-10A7-41A0-A85E-EE6D0E267367}.Debug|x86.ActiveCfg = Debug|Any CPU + {E0F934F9-10A7-41A0-A85E-EE6D0E267367}.Debug|x86.Build.0 = Debug|Any CPU + {E0F934F9-10A7-41A0-A85E-EE6D0E267367}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E0F934F9-10A7-41A0-A85E-EE6D0E267367}.Release|Any CPU.Build.0 = Release|Any CPU + {E0F934F9-10A7-41A0-A85E-EE6D0E267367}.Release|x64.ActiveCfg = Release|Any CPU + {E0F934F9-10A7-41A0-A85E-EE6D0E267367}.Release|x64.Build.0 = Release|Any CPU + {E0F934F9-10A7-41A0-A85E-EE6D0E267367}.Release|x86.ActiveCfg = Release|Any CPU + {E0F934F9-10A7-41A0-A85E-EE6D0E267367}.Release|x86.Build.0 = Release|Any CPU {3FA7E5DF-03B7-4586-A980-85C155B376C5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {3FA7E5DF-03B7-4586-A980-85C155B376C5}.Debug|Any CPU.Build.0 = Debug|Any CPU {3FA7E5DF-03B7-4586-A980-85C155B376C5}.Debug|x64.ActiveCfg = Debug|Any CPU @@ -805,6 +819,7 @@ Global EndGlobalSection GlobalSection(NestedProjects) = preSolution {7AECC7C6-9A21-4B8A-84D9-AFC4F5840CAF} = {1A647B41-53D0-4638-AE5A-6630BAAE45FC} + {E0F934F9-10A7-41A0-A85E-EE6D0E267367} = {1A647B41-53D0-4638-AE5A-6630BAAE45FC} {E431D279-E02B-4670-B934-3DB9F15D8CCC} = {1A647B41-53D0-4638-AE5A-6630BAAE45FC} {31EC2647-6A5A-42D1-B7B5-02804B340726} = {E431D279-E02B-4670-B934-3DB9F15D8CCC} {AFFA4143-DC28-4FBE-A33B-D6414F541EA4} = {E431D279-E02B-4670-B934-3DB9F15D8CCC} diff --git a/nuget.config b/nuget.config new file mode 100644 index 0000000..dddb33b --- /dev/null +++ b/nuget.config @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + diff --git a/src/Gcp/CloudRun/OpenTelemetry/.gitignore b/src/Gcp/CloudRun/OpenTelemetry/.gitignore new file mode 100644 index 0000000..f2850e0 --- /dev/null +++ b/src/Gcp/CloudRun/OpenTelemetry/.gitignore @@ -0,0 +1,3 @@ +# Temporary test scaffolding for the unpublished GCP Cloud Run package (remove once published). +local-packages/ +nuget.config diff --git a/src/Gcp/CloudRun/OpenTelemetry/CloudRunWorkerSample.cs b/src/Gcp/CloudRun/OpenTelemetry/CloudRunWorkerSample.cs new file mode 100644 index 0000000..502dd26 --- /dev/null +++ b/src/Gcp/CloudRun/OpenTelemetry/CloudRunWorkerSample.cs @@ -0,0 +1,20 @@ +namespace TemporalioSamples.Gcp.CloudRun.OpenTelemetry; + +using Temporalio.Worker; + +/// +/// Shared worker configuration so both the entrypoint and the tests register the same +/// workflow and activities. +/// +public static class CloudRunWorkerSample +{ + /// + /// Register the sample workflow and activities on the given worker options. + /// + /// Worker options to configure. + /// The same options, for chaining. + public static TemporalWorkerOptions ConfigureOptions(TemporalWorkerOptions options) => + options. + AddWorkflow(). + AddActivity(GreetingActivities.SayHello); +} diff --git a/src/Gcp/CloudRun/OpenTelemetry/Dockerfile b/src/Gcp/CloudRun/OpenTelemetry/Dockerfile new file mode 100644 index 0000000..4d2037b --- /dev/null +++ b/src/Gcp/CloudRun/OpenTelemetry/Dockerfile @@ -0,0 +1,22 @@ +# syntax=docker/dockerfile:1 +# +# Builds the Cloud Run worker image. Temporalio (with its bundled native bridge for linux) is +# restored from NuGet; the GCP Cloud Run OpenTelemetry package is restored from the local folder +# feed under this sample until it is published (see nuget.config / local-packages). No SDK-from- +# source or Rust build is needed. +# +# Build context is the samples-dotnet repo root (so the shared Directory.*.props / global.json are +# available): +# docker build -f src/Gcp/CloudRun/OpenTelemetry/Dockerfile -t . +FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build +WORKDIR /src +COPY global.json Directory.Build.props Directory.Packages.props .editorconfig ./ +COPY src/Gcp/CloudRun/OpenTelemetry/ ./src/Gcp/CloudRun/OpenTelemetry/ +RUN dotnet publish src/Gcp/CloudRun/OpenTelemetry/TemporalioSamples.Gcp.CloudRun.OpenTelemetry.csproj -c Release -o /app + +FROM mcr.microsoft.com/dotnet/runtime:8.0 +RUN useradd --create-home --uid 10001 worker +WORKDIR /app +COPY --from=build --chown=worker:worker /app ./ +USER 10001 +ENTRYPOINT ["dotnet", "TemporalioSamples.Gcp.CloudRun.OpenTelemetry.dll"] diff --git a/src/Gcp/CloudRun/OpenTelemetry/GreetingActivities.cs b/src/Gcp/CloudRun/OpenTelemetry/GreetingActivities.cs new file mode 100644 index 0000000..a04bf65 --- /dev/null +++ b/src/Gcp/CloudRun/OpenTelemetry/GreetingActivities.cs @@ -0,0 +1,14 @@ +namespace TemporalioSamples.Gcp.CloudRun.OpenTelemetry; + +using Microsoft.Extensions.Logging; +using Temporalio.Activities; + +public static class GreetingActivities +{ + [Activity] + public static string SayHello(string name) + { + ActivityExecutionContext.Current.Logger.LogInformation("SayHello activity: {Name}", name); + return $"Hello, {name}!"; + } +} diff --git a/src/Gcp/CloudRun/OpenTelemetry/GreetingWorkflow.workflow.cs b/src/Gcp/CloudRun/OpenTelemetry/GreetingWorkflow.workflow.cs new file mode 100644 index 0000000..98e4463 --- /dev/null +++ b/src/Gcp/CloudRun/OpenTelemetry/GreetingWorkflow.workflow.cs @@ -0,0 +1,19 @@ +namespace TemporalioSamples.Gcp.CloudRun.OpenTelemetry; + +using Microsoft.Extensions.Logging; +using Temporalio.Workflows; + +[Workflow] +public class GreetingWorkflow +{ + [WorkflowRun] + public async Task RunAsync(string name) + { + Workflow.Logger.LogInformation("GreetingWorkflow started: {Name}", name); + var result = await Workflow.ExecuteActivityAsync( + () => GreetingActivities.SayHello(name), + new() { StartToCloseTimeout = TimeSpan.FromSeconds(10) }); + Workflow.Logger.LogInformation("GreetingWorkflow completed: {Result}", result); + return result; + } +} diff --git a/src/Gcp/CloudRun/OpenTelemetry/Program.cs b/src/Gcp/CloudRun/OpenTelemetry/Program.cs new file mode 100644 index 0000000..c8a5856 --- /dev/null +++ b/src/Gcp/CloudRun/OpenTelemetry/Program.cs @@ -0,0 +1,75 @@ +using System.Runtime.InteropServices; +using Microsoft.Extensions.Logging; +using Temporalio.Client; +using Temporalio.Common.EnvConfig; +using Temporalio.Extensions.Gcp.CloudRun.OpenTelemetry; +using Temporalio.Worker; +using TemporalioSamples.Gcp.CloudRun.OpenTelemetry; + +// Build client connection options from environment configuration (TEMPORAL_ADDRESS, +// TEMPORAL_NAMESPACE, TEMPORAL_API_KEY, ...). With no API key and no TLS block this connects in +// plaintext, which is what a local dev server (reached over an ngrok TCP tunnel) needs. +var connectOptions = ClientEnvConfig.LoadClientConnectOptions(); +connectOptions.TargetHost ??= "localhost:7233"; + +// Send all Temporal logs to stdout so Cloud Run captures them in Cloud Logging. +connectOptions.LoggerFactory = LoggerFactory.Create(builder => + builder. + AddSimpleConsole(options => options.TimestampFormat = "[HH:mm:ss] "). + SetMinimumLevel(LogLevel.Information)); + +var taskQueue = Environment.GetEnvironmentVariable("TEMPORAL_TASK_QUEUE") ?? "cloud-run-worker"; + +// The --starter mode runs a single workflow (useful for kicking off work locally against the same +// server the deployed worker polls). Applying the defaults here too propagates a trace context into +// the workflow so the deployed worker's spans join the same distributed trace. +if (args.Contains("--starter")) +{ + using var starterTelemetry = connectOptions.ApplyGoogleCloudRunOpenTelemetryDefaults(); + var starterClient = await TemporalClient.ConnectAsync(connectOptions); + var greeting = await starterClient.ExecuteWorkflowAsync( + (GreetingWorkflow wf) => wf.RunAsync("Temporal"), + new($"cloud-run-worker-{Guid.NewGuid():N}", taskQueue)); + Console.WriteLine("Workflow result: {0}", greeting); + await starterTelemetry.FlushAsync(TimeSpan.FromSeconds(2)); + return; +} + +// Apply the Google Cloud Run OpenTelemetry defaults: adds the tracing interceptor and configures a +// Temporal runtime that exports Core metrics + traces over OTLP to the local collector sidecar. The +// returned handle owns the tracer provider and is flushed on shutdown. +using var telemetry = connectOptions.ApplyGoogleCloudRunOpenTelemetryDefaults(); + +var client = await TemporalClient.ConnectAsync(connectOptions); + +using var cts = new CancellationTokenSource(); +Console.CancelKeyPress += (_, eventArgs) => +{ + eventArgs.Cancel = true; + cts.Cancel(); +}; + +// Cloud Run signals shutdown with SIGTERM (about 10 seconds before SIGKILL). +using var sigterm = PosixSignalRegistration.Create(PosixSignal.SIGTERM, _ => cts.Cancel()); + +using var worker = new TemporalWorker( + client, CloudRunWorkerSample.ConfigureOptions(new(taskQueue))); + +Console.WriteLine( + "Worker running: taskQueue={0} address={1} namespace={2}", + taskQueue, + connectOptions.TargetHost, + connectOptions.Namespace ?? "default"); +try +{ + await worker.ExecuteAsync(cts.Token); +} +catch (OperationCanceledException) +{ + Console.WriteLine("Worker shutting down"); +} + +// Flush buffered traces within the Cloud Run shutdown grace window. Core metrics are exported +// periodically by the runtime and have no explicit flush. +await telemetry.FlushAsync(TimeSpan.FromSeconds(2)); +Console.WriteLine("Worker stopped"); diff --git a/src/Gcp/CloudRun/OpenTelemetry/TemporalioSamples.Gcp.CloudRun.OpenTelemetry.csproj b/src/Gcp/CloudRun/OpenTelemetry/TemporalioSamples.Gcp.CloudRun.OpenTelemetry.csproj new file mode 100644 index 0000000..bec8e3e --- /dev/null +++ b/src/Gcp/CloudRun/OpenTelemetry/TemporalioSamples.Gcp.CloudRun.OpenTelemetry.csproj @@ -0,0 +1,11 @@ + + + + Exe + + + + + + + diff --git a/src/Gcp/CloudRun/OpenTelemetry/collector-config.yaml b/src/Gcp/CloudRun/OpenTelemetry/collector-config.yaml new file mode 100644 index 0000000..2fa7df6 --- /dev/null +++ b/src/Gcp/CloudRun/OpenTelemetry/collector-config.yaml @@ -0,0 +1,88 @@ +# Google-Built OpenTelemetry Collector config for the Cloud Run worker-pool sidecar. +# metrics -> Google Managed Service for Prometheus (googlemanagedprometheus) +# traces -> Cloud Trace via the Telemetry API (OTLP), authenticated with the runtime SA (ADC) +# The worker exports OTLP/gRPC to localhost:4317; the collector detects GCP resource attributes and +# fans out. Auth uses the worker-pool service account's Application Default Credentials via the +# googleclientauth extension (no key files). +receivers: + otlp: + protocols: + grpc: + endpoint: localhost:4317 + +processors: + # Batch traces for throughput. Do NOT add a batch processor to the cumulative-metrics pipeline: a + # shutdown flush could be batched with a recent periodic export of the same series and rejected as + # a duplicate time series. + batch/traces: + send_batch_max_size: 200 + send_batch_size: 200 + timeout: 5s + memory_limiter: + check_interval: 1s + limit_percentage: 65 + spike_limit_percentage: 20 + resourcedetection: + detectors: [gcp] + timeout: 10s + # Rename Temporal datapoint labels that collide with the target labels Google Managed Service for + # Prometheus injects (e.g. Temporal emits a `namespace` label). + transform/collision: + metric_statements: + - context: datapoint + statements: + - set(attributes["exported_location"], attributes["location"]) + - delete_key(attributes, "location") + - set(attributes["exported_cluster"], attributes["cluster"]) + - delete_key(attributes, "cluster") + - set(attributes["exported_namespace"], attributes["namespace"]) + - delete_key(attributes, "namespace") + - set(attributes["exported_job"], attributes["job"]) + - delete_key(attributes, "job") + - set(attributes["exported_instance"], attributes["instance"]) + - delete_key(attributes, "instance") + - set(attributes["exported_project_id"], attributes["project_id"]) + - delete_key(attributes, "project_id") + # The Telemetry API expects the Google Cloud project in gcp.project_id. + transform/set_project_id: + error_mode: ignore + trace_statements: + - set(resource.attributes["gcp.project_id"], resource.attributes["gcp.project.id"]) where resource.attributes["gcp.project.id"] != nil + - set(resource.attributes["gcp.project_id"], resource.attributes["cloud.account.id"]) where resource.attributes["gcp.project_id"] == nil and resource.attributes["cloud.account.id"] != nil + +exporters: + googlemanagedprometheus: + otlp: + endpoint: telemetry.googleapis.com:443 + compression: none + balancer_name: pick_first + auth: + authenticator: googleclientauth + +extensions: + health_check: + endpoint: 0.0.0.0:13133 + googleclientauth: + +service: + extensions: + - health_check + - googleclientauth + pipelines: + metrics: + receivers: [otlp] + processors: [memory_limiter, resourcedetection, transform/collision] + exporters: [googlemanagedprometheus] + traces: + receivers: [otlp] + processors: [memory_limiter, resourcedetection, transform/set_project_id, batch/traces] + exporters: [otlp] + telemetry: + metrics: + readers: + - periodic: + exporter: + otlp: + protocol: grpc + endpoint: http://localhost:4317 + insecure: true diff --git a/src/Gcp/CloudRun/OpenTelemetry/worker-pool.yaml b/src/Gcp/CloudRun/OpenTelemetry/worker-pool.yaml new file mode 100644 index 0000000..8dd3d69 --- /dev/null +++ b/src/Gcp/CloudRun/OpenTelemetry/worker-pool.yaml @@ -0,0 +1,62 @@ +# Cloud Run WorkerPool: a continuously-polling Temporal worker + a Google-Built OpenTelemetry +# Collector sidecar. Render placeholders with `envsubst` (see README) then apply with +# `gcloud run worker-pools replace`. +# +# This sample connects to a Temporal server over TEMPORAL_ADDRESS with no API key / no TLS (e.g. a +# local dev server exposed via an ngrok TCP tunnel). For Temporal Cloud, add a TEMPORAL_API_KEY env +# var sourced from a Secret Manager secretKeyRef (see the Java/Python samples). +apiVersion: run.googleapis.com/v1 +kind: WorkerPool +metadata: + name: "${WORKER_POOL}" + labels: + cloud.googleapis.com/location: "${REGION}" + annotations: + run.googleapis.com/scalingMode: manual + run.googleapis.com/manualInstanceCount: "${INSTANCE_COUNT}" +spec: + template: + metadata: + annotations: + run.googleapis.com/container-dependencies: '{"worker":["collector"]}' + run.googleapis.com/execution-environment: gen2 + spec: + containerConcurrency: 0 + serviceAccountName: "${SERVICE_ACCOUNT_EMAIL}" + containers: + - name: worker + image: "${WORKER_IMAGE}" + env: + - name: TEMPORAL_ADDRESS + value: "${TEMPORAL_ADDRESS}" + - name: TEMPORAL_NAMESPACE + value: "${TEMPORAL_NAMESPACE}" + - name: TEMPORAL_TASK_QUEUE + value: "${TEMPORAL_TASK_QUEUE}" + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: http://localhost:4317 + resources: + limits: + cpu: "1" + memory: 512Mi + - name: collector + image: us-docker.pkg.dev/cloud-ops-agents-artifacts/google-cloud-opentelemetry-collector/otelcol-google:0.156.0 + args: + - --config=env:OTELCOL_CONFIG + env: + - name: OTELCOL_CONFIG + valueFrom: + secretKeyRef: + key: "${COLLECTOR_CONFIG_SECRET_VERSION}" + name: "${COLLECTOR_CONFIG_SECRET}" + startupProbe: + httpGet: + path: / + port: 13133 + timeoutSeconds: 1 + periodSeconds: 2 + failureThreshold: 30 + resources: + limits: + cpu: "1" + memory: 512Mi diff --git a/tests/Gcp/CloudRun/OpenTelemetry/CloudRunWorkerTests.cs b/tests/Gcp/CloudRun/OpenTelemetry/CloudRunWorkerTests.cs new file mode 100644 index 0000000..a24889c --- /dev/null +++ b/tests/Gcp/CloudRun/OpenTelemetry/CloudRunWorkerTests.cs @@ -0,0 +1,33 @@ +namespace TemporalioSamples.Tests.Gcp.CloudRun.OpenTelemetry; + +using Temporalio.Client; +using Temporalio.Testing; +using Temporalio.Worker; +using TemporalioSamples.Gcp.CloudRun.OpenTelemetry; +using Xunit; +using Xunit.Abstractions; + +public class CloudRunWorkerTests : TestBase +{ + public CloudRunWorkerTests(ITestOutputHelper output) + : base(output) + { + } + + [TimeSkippingServerFact] + public async Task GreetingWorkflow_SimpleRun_Succeeds() + { + await using var env = await WorkflowEnvironment.StartTimeSkippingAsync(); + using var worker = new TemporalWorker( + env.Client, + CloudRunWorkerSample.ConfigureOptions( + new TemporalWorkerOptions("cloud-run-worker-test-task-queue"))); + await worker.ExecuteAsync(async () => + { + var result = await env.Client.ExecuteWorkflowAsync( + (GreetingWorkflow wf) => wf.RunAsync("Cloud Run"), + new(id: $"workflow-{Guid.NewGuid()}", taskQueue: worker.Options.TaskQueue!)); + Assert.Equal("Hello, Cloud Run!", result); + }); + } +} diff --git a/tests/TemporalioSamples.Tests.csproj b/tests/TemporalioSamples.Tests.csproj index 234002a..dee6bd5 100644 --- a/tests/TemporalioSamples.Tests.csproj +++ b/tests/TemporalioSamples.Tests.csproj @@ -22,6 +22,7 @@ +