From d3866188c405b9b533df090db70ce16080f205a8 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Thu, 2 Jul 2026 14:53:12 -0400 Subject: [PATCH 1/2] spec: scaffold for DSPX-3635 otelo --- spec/DSPX-3635.md | 63 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 spec/DSPX-3635.md diff --git a/spec/DSPX-3635.md b/spec/DSPX-3635.md new file mode 100644 index 0000000000..1afc53a241 --- /dev/null +++ b/spec/DSPX-3635.md @@ -0,0 +1,63 @@ +--- +ticket: DSPX-3635 +title: End-to-end OpenTelemetry tracing: xtest → SDK CLIs → platform/KAS +status: draft +authors: [dmihalcik@virtru.com] +branches: [opentdf/platform:DSPX-3635-otelo, opentdf/tests:DSPX-3635-otelo, opentdf/java-sdk:DSPX-3635-otelo, opentdf/web-sdk:DSPX-3635-otelo] +prs: [] +created: 2026-07-02 +updated: 2026-07-02 +--- + +# End-to-end OpenTelemetry tracing: xtest → SDK CLIs → platform/KAS + +## Summary +ProblemDebugging xtest failures today means correlating log lines across seven services (platform + 6 KAS instances) by timestamp and best guesses. There is no way to take a failing test and pull up "the trace" — server-side OTel spans exist (service/tracing) but no client carries a traceparent header in, so each request becomes its own root. +This proposal threads a single trace from the pytest test body, through the SDK CLI (cli.sh encrypt/decrypt), into platform/KAS, and exports everything to a local Jaeger so failures link directly to a UI URL. +Architecturepytest test ──► cli.sh ──► platform/KAS + span: test.X span: cli.encrypt spans: rewrap, GetDecision, ... + │ TRACEPARENT env │ traceparent header │ existing service/tracing + └──────────────────┴──────────────────────┴─► OTLP collector ─► Jaeger UIScope (phased)Infra: otdf-local up optionally starts a Jaeger all-in-one container (OTLP 4317, UI 16686). Platform/KAS already export via service/tracing; just need OTEL_EXPORTER_OTLP_ENDPOINT wired in. +Pytest plugin (xtest/fixtures/tracing.py): +Session-scoped tracer init, service name xtest. +Autouse function fixture wraps each test in a span (pytest.test, attrs: test.name, test.sdk, test.container), exports TRACEPARENT into env for CLI subprocess invocations. +On test failure, print Trace: http://localhost:16686/trace/{trace_id} to captured stdout so the failure summary links directly. +Go CLI (otdfctl + xtest/sdk/go shim) — proof of concept: +Read TRACEPARENT and OTEL_EXPORTER_OTLP_ENDPOINT env at startup. +Init OTel tracer w/ otlpgrpc exporter; root span per command (cli.encrypt, cli.decrypt). +Use otelgrpc.NewClientHandler on platform/KAS connections so spans propagate. +Java SDK — io.opentelemetry.instrumentation:opentelemetry-grpc-1.6, same pattern. +JS SDK — @opentelemetry/sdk-trace-node + @opentelemetry/instrumentation-fetch for the Node CLI shim. Browser/web-sdk in-browser tracing deferred to a separate ticket. +slog correlation — wrap the platform's slog handler to inject trace_id and span_id from the active span context (partly done in service/tracing — verify, finish). +Non-goalsMetrics (this is traces + log correlation only). +Production deploy story / persistent backend. +Browser-side tracing for vulnerability/ Playwright suite. +Risks / constraintsEvery SDK CLI must no-op cleanly when OTEL_EXPORTER_OTLP_ENDPOINT is unset — no extra latency, no startup failures. +JS SDK adds a few hundred KB; gate behind env var so non-CI users don't pay it. +Jaeger all-in-one adds ~200MB RAM in otdf-local up; make optional via flag (otdf-local up --tracing). +Each SDK's CLI shim picks up a new direct dep — keep the OTel surface area minimal (manual init, no auto-instrumentation agents). +Success criteriaRunning pytest --sdks go test_tdfs.py -v with --tracing (or env var) prints a Jaeger trace URL per test. +That URL shows the full chain: pytest.test → cli.encrypt → platform Rewrap → KAS Rewrap → GetDecision. +Platform/KAS log lines emitted during the test include matching trace_id/span_id fields. +All four CLI paths (go, java, js Node, plus pytest) hit the same trace tree. +Suggested rollout orderLand 1+2+3 first (infra, pytest plugin, Go CLI) — minimum viable end-to-end loop. Java and JS follow once the pattern is proven. slog correlation can run in parallel with any of the SDK phases. +ContextCame out of the DSPX-3397 DPoP debugging session — chasing PDP denials across the platform + 6 KAS log files with no per-request correlation was the dominant time sink. A working trace would have collapsed that diagnostic loop from hours to minutes. + +## Problem / Motivation +_Why does this work need to happen? What is the user/business pain?_ + +## Proposed Solution +_What will you build, at a functional level? Sketch the approach._ + +## Inputs / Outputs / Contracts +_Function signatures, data shapes, API contracts, CLI flags._ + +## Edge Cases & Constraints +_Boundary conditions, error states, performance limits, security considerations._ + +## Out of Scope +_What this work item explicitly does not cover._ + +## Acceptance Criteria +- [ ] _Clear, testable condition_ +- [ ] _…_ From 846e983f69749cbad38eb2533da7b8a1fcdf38b6 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Thu, 2 Jul 2026 23:53:20 -0400 Subject: [PATCH 2/2] feat(otel): propagate traces through otdfctl and correlate platform logs Thread end-to-end OpenTelemetry tracing through the Go CLI and add trace/log correlation on the platform, per DSPX-3635. - otdfctl: new pkg/tracing (no-op unless OTEL_EXPORTER_OTLP_ENDPOINT is set), a root span per command that continues TRACEPARENT from the environment, and an otelconnect client interceptor via sdk.WithExtraClientOptions so platform/KAS calls carry the trace. Context is threaded into CreateTDFContext (encrypt) and Reader.Init (decrypt) so the KAS Rewrap joins the trace; a cli.OnExit hook flushes spans on os.Exit paths. - service/logger: ContextHandler stamps trace_id/span_id from the active span onto every log record. Signed-off-by: Dave Mihalcik --- otdfctl/cmd/execute.go | 2 + otdfctl/cmd/root.go | 34 +++++++++ otdfctl/cmd/tdf/encrypt.go | 1 + otdfctl/go.mod | 14 +++- otdfctl/go.sum | 20 ++++- otdfctl/pkg/cli/errors.go | 15 ++++ otdfctl/pkg/handlers/sdk.go | 12 +++ otdfctl/pkg/handlers/tdf.go | 10 ++- otdfctl/pkg/tracing/tracing.go | 106 ++++++++++++++++++++++++++ otdfctl/pkg/tracing/tracing_test.go | 63 +++++++++++++++ service/logger/contextHandler.go | 11 +++ service/logger/contextHandler_test.go | 64 ++++++++++++++++ 12 files changed, 346 insertions(+), 6 deletions(-) create mode 100644 otdfctl/pkg/tracing/tracing.go create mode 100644 otdfctl/pkg/tracing/tracing_test.go create mode 100644 service/logger/contextHandler_test.go diff --git a/otdfctl/cmd/execute.go b/otdfctl/cmd/execute.go index 32dc8ec7c3..0519271462 100644 --- a/otdfctl/cmd/execute.go +++ b/otdfctl/cmd/execute.go @@ -40,11 +40,13 @@ func Execute(opts ...ExecuteOptFunc) { if c.mountTo != nil { err := MountRoot(c.mountTo, c.renameCmd) if err != nil { + finishTrace() os.Exit(cli.ExitCodeError) } } else { err := RootCmd.Execute() if err != nil { + finishTrace() os.Exit(cli.ExitCodeError) } } diff --git a/otdfctl/cmd/root.go b/otdfctl/cmd/root.go index 89ab6f9fcb..aa9350fee2 100644 --- a/otdfctl/cmd/root.go +++ b/otdfctl/cmd/root.go @@ -1,9 +1,11 @@ package cmd import ( + "context" "fmt" "log/slog" "os" + "sync" "github.com/opentdf/platform/otdfctl/cmd/auth" cfg "github.com/opentdf/platform/otdfctl/cmd/config" @@ -14,8 +16,10 @@ import ( "github.com/opentdf/platform/otdfctl/pkg/cli" "github.com/opentdf/platform/otdfctl/pkg/config" "github.com/opentdf/platform/otdfctl/pkg/man" + "github.com/opentdf/platform/otdfctl/pkg/tracing" "github.com/opentdf/platform/sdk" "github.com/spf13/cobra" + "go.opentelemetry.io/otel" ) var ( @@ -23,6 +27,11 @@ var ( clientCredsJSON string RootCmd = &man.Docs.GetDoc("").Command + + // finishTrace ends the root command span and flushes the tracer provider. + // It is invoked from PersistentPostRunE (normal exit) and via cli.OnExit + // (os.Exit paths), guarded so it runs exactly once. + finishTrace = func() {} ) type version struct { @@ -86,6 +95,31 @@ func init() { slog.SetDefault(logger) } + + // Start OpenTelemetry tracing for this command. This is a no-op unless + // OTEL_EXPORTER_OTLP_ENDPOINT is set, so it adds no cost to normal runs. + shutdown, enabled := tracing.Init(context.Background()) + if enabled { + ctx := tracing.ExtractParentFromEnv(context.Background()) + ctx, span := otel.Tracer("otdfctl").Start(ctx, "cli."+cmd.Name()) + cmd.SetContext(ctx) + + var once sync.Once + finishTrace = func() { + once.Do(func() { + span.End() + shutdown() + }) + } + // Ensure spans flush even when a command exits via os.Exit. + cli.OnExit = finishTrace + } + return nil + } + + // Flush traces on the normal (non-os.Exit) return path. + RootCmd.PersistentPostRunE = func(_ *cobra.Command, _ []string) error { + finishTrace() return nil } diff --git a/otdfctl/cmd/tdf/encrypt.go b/otdfctl/cmd/tdf/encrypt.go index 3935825f28..bbd21dce21 100644 --- a/otdfctl/cmd/tdf/encrypt.go +++ b/otdfctl/cmd/tdf/encrypt.go @@ -108,6 +108,7 @@ func encryptRun(cmd *cobra.Command, args []string) { // Do the encryption encrypted, err := h.EncryptBytes( + c.Context(), tdfType, bytesSlice, attrValues, diff --git a/otdfctl/go.mod b/otdfctl/go.mod index 37f646d54d..1de05f8d70 100644 --- a/otdfctl/go.mod +++ b/otdfctl/go.mod @@ -3,6 +3,8 @@ module github.com/opentdf/platform/otdfctl go 1.25.0 require ( + connectrpc.com/connect v1.20.0 + connectrpc.com/otelconnect v0.9.0 github.com/adrg/frontmatter v0.2.0 github.com/charmbracelet/bubbles v0.21.1-0.20250623103423-23b8fd6302d7 github.com/charmbracelet/bubbletea v1.3.10 @@ -23,6 +25,9 @@ require ( github.com/spf13/cobra v1.10.2 github.com/stretchr/testify v1.11.1 github.com/zitadel/oidc/v3 v3.45.1 + go.opentelemetry.io/otel v1.43.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.43.0 + go.opentelemetry.io/otel/sdk v1.43.0 golang.org/x/oauth2 v0.36.0 golang.org/x/term v0.43.0 google.golang.org/grpc v1.81.1 @@ -32,7 +37,6 @@ require ( require ( al.essio.dev/pkg/shellescape v1.5.1 // indirect buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260415201107-50325440f8f2.1 // indirect - connectrpc.com/connect v1.20.0 // indirect github.com/BurntSushi/toml v0.3.1 // indirect github.com/Masterminds/semver/v3 v3.5.0 // indirect github.com/alecthomas/chroma/v2 v2.14.0 // indirect @@ -40,6 +44,7 @@ require ( github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/catppuccin/go v0.3.0 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect github.com/charmbracelet/x/ansi v0.10.1 // indirect @@ -63,6 +68,7 @@ require ( github.com/gorilla/securecookie v1.1.2 // indirect github.com/gowebpki/jcs v1.0.1 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/lestrrat-go/blackmagic v1.0.4 // indirect github.com/lestrrat-go/httpcc v1.0.1 // indirect @@ -97,15 +103,17 @@ require ( github.com/zitadel/logging v0.6.2 // indirect github.com/zitadel/schema v1.3.1 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/otel v1.43.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 // indirect go.opentelemetry.io/otel/metric v1.43.0 // indirect go.opentelemetry.io/otel/trace v1.43.0 // indirect + go.opentelemetry.io/proto/otlp v1.10.0 // indirect golang.org/x/crypto v0.52.0 // indirect golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 // indirect golang.org/x/net v0.54.0 // indirect golang.org/x/sys v0.45.0 // indirect golang.org/x/text v0.37.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/otdfctl/go.sum b/otdfctl/go.sum index 2f49fe6e1f..520b98f736 100644 --- a/otdfctl/go.sum +++ b/otdfctl/go.sum @@ -6,6 +6,8 @@ connectrpc.com/connect v1.20.0 h1:6TNDAB+WeNd2uolWNlYczB5E0KNNaVMNUEx8JEUsPmQ= connectrpc.com/connect v1.20.0/go.mod h1:A2ygJrukXwWy32vkCAAHNVguZrqZ+jeZ9rGRnGR4dN4= connectrpc.com/grpchealth v1.4.0 h1:MJC96JLelARPgZTiRF9KRfY/2N9OcoQvF2EWX07v2IE= connectrpc.com/grpchealth v1.4.0/go.mod h1:WhW6m1EzTmq3Ky1FE8EfkIpSDc6TfUx2M2KqZO3ts/Q= +connectrpc.com/otelconnect v0.9.0 h1:NggB3pzRC3pukQWaYbRHJulxuXvmCKCKkQ9hbrHAWoA= +connectrpc.com/otelconnect v0.9.0/go.mod h1:AEkVLjCPXra+ObGFCOClcJkNjS7zPaQSqvO0lCyjfZc= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= @@ -32,6 +34,8 @@ github.com/bmatcuk/doublestar/v4 v4.9.1 h1:X8jg9rRZmJd4yRy7ZeNDRnM+T3ZfHv15JiBJ/ github.com/bmatcuk/doublestar/v4 v4.9.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/catppuccin/go v0.3.0 h1:d+0/YicIq+hSTo5oPuRi5kOpqkVA5tAsU6dNhvRu+aY= github.com/catppuccin/go v0.3.0/go.mod h1:8IHJuMGaUUjQM82qBrGNBv7LFq6JI3NnQCF6MOlZjpc= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/bubbles v0.21.1-0.20250623103423-23b8fd6302d7 h1:JFgG/xnwFfbezlUnFMJy0nusZvytYysV4SCS2cYbvws= @@ -125,6 +129,8 @@ github.com/gowebpki/jcs v1.0.1 h1:Qjzg8EOkrOTuWP7DqQ1FbYtcpEbeTzUoTN9bptp8FOU= github.com/gowebpki/jcs v1.0.1/go.mod h1:CID1cNZ+sHp1CCpAR8mPf6QRtagFBgPJE0FCUQ6+BrI= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3 h1:B+8ClL/kCQkRiU82d9xajRPKYMrB7E0MbtzWVi1K4ns= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3/go.mod h1:NbCUVmiS4foBGBHOYlCT25+YmGpJ32dZPi75pGEUpj4= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -244,6 +250,10 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 h1:88Y4s2C8oTui1LGM6bTWkw0ICGcOLCAI5l6zsD1j20k= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0/go.mod h1:Vl1/iaggsuRlrHf/hfPJPvVag77kKyvrLeD10kpMl+A= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.43.0 h1:RAE+JPfvEmvy+0LzyUA25/SGawPwIUbZ6u0Wug54sLc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.43.0/go.mod h1:AGmbycVGEsRx9mXMZ75CsOyhSP6MFIcj/6dnG+vhVjk= go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= @@ -252,6 +262,10 @@ go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfC go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= +go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -309,8 +323,10 @@ golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 h1:ggcbiqK8WWh6l1dnltU4BgWGIGo+EVYxCaAPih/zQXQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 h1:VPWxll4HlMw1Vs/qXtN7BvhZqsS9cdAittCNvVENElA= +google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:7QBABkRtR8z+TEnmXTqIqwJLlzrZKVfAUm7tY3yGv0M= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 h1:m8qni9SQFH0tJc1X0vmnpw/0t+AImlSvp30sEupozUg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ= google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= diff --git a/otdfctl/pkg/cli/errors.go b/otdfctl/pkg/cli/errors.go index 4d79d9cc4e..8804d595af 100644 --- a/otdfctl/pkg/cli/errors.go +++ b/otdfctl/pkg/cli/errors.go @@ -14,6 +14,18 @@ const ( ExitCodeError = 1 ) +// OnExit, if set, is invoked immediately before the process exits via any of +// the Exit* helpers. It gives long-lived resources (e.g. an OpenTelemetry +// tracer provider) a chance to flush, since os.Exit does not run deferred +// functions. It must be safe to call more than once. +var OnExit func() + +func runOnExit() { + if OnExit != nil { + OnExit() + } +} + func ExitWithError(errMsg string, err error) { // This is temporary until we can refactor the code to use the Cli struct (&Cli{printer: defaultPrinter()}).ExitWithError(errMsg, err) @@ -67,12 +79,14 @@ func (c *Cli) ExitWithMessage(msg string, code int) { } else { c.println(w, msg) } + runOnExit() os.Exit(code) } func (c *Cli) ExitWithJSON(v interface{}, code int) { if c.printer.json { c.printJSON(v, os.Stdout) + runOnExit() os.Exit(code) } } @@ -85,6 +99,7 @@ func (c *Cli) ExitWith(styledMsg string, jsonMsg interface{}, code int, w io.Wri } else { c.println(w, styledMsg) } + runOnExit() os.Exit(code) } diff --git a/otdfctl/pkg/handlers/sdk.go b/otdfctl/pkg/handlers/sdk.go index 2c1e0f7871..01cffb88ae 100644 --- a/otdfctl/pkg/handlers/sdk.go +++ b/otdfctl/pkg/handlers/sdk.go @@ -4,6 +4,8 @@ import ( "errors" "log/slog" + "connectrpc.com/connect" + "connectrpc.com/otelconnect" "github.com/opentdf/platform/otdfctl/pkg/auth" "github.com/opentdf/platform/otdfctl/pkg/profiles" "github.com/opentdf/platform/otdfctl/pkg/utils" @@ -88,6 +90,16 @@ func New(opts ...handlerOptsFunc) (Handler, error) { sdk.WithConnectionValidation(), sdk.WithLogger(slog.Default()), } + + // Propagate the active trace context to platform/KAS so their spans join + // the CLI's trace. When no tracer provider is configured this uses the + // global no-op provider and is effectively inert. + if otelInterceptor, err := otelconnect.NewInterceptor(otelconnect.WithoutTraceEvents()); err != nil { + slog.Warn("failed to create otel connect interceptor; tracing disabled", slog.Any("error", err)) + } else { + defaultSDKOpts = append(defaultSDKOpts, sdk.WithExtraClientOptions(connect.WithInterceptors(otelInterceptor))) + } + if o.TLSNoVerify { defaultSDKOpts = append(defaultSDKOpts, sdk.WithInsecureSkipVerifyConn()) } diff --git a/otdfctl/pkg/handlers/tdf.go b/otdfctl/pkg/handlers/tdf.go index d8a5b0d69f..dac1e4e7a2 100644 --- a/otdfctl/pkg/handlers/tdf.go +++ b/otdfctl/pkg/handlers/tdf.go @@ -39,6 +39,7 @@ type TDFInspect struct { } func (h Handler) EncryptBytes( + ctx context.Context, tdfType string, unencrypted []byte, attrValues []string, @@ -94,7 +95,7 @@ func (h Handler) EncryptBytes( opts = append(opts, sdk.WithTargetMode(targetMode)) } - _, err := h.sdk.CreateTDF(enc, bytes.NewReader(unencrypted), opts...) + _, err := h.sdk.CreateTDFContext(ctx, enc, bytes.NewReader(unencrypted), opts...) return enc, err default: return nil, errors.New("unknown TDF type") @@ -149,6 +150,13 @@ func (h Handler) DecryptBytes( if err != nil { return nil, err } + // Perform the key unwrap (KAS Rewrap) up front with the caller's + // context so the request propagates the active trace. Without this the + // unwrap is triggered lazily by io.Copy below using context.Background, + // which would start a new, disconnected trace. + if err = r.Init(ctx); err != nil { + return nil, formatDecryptError(ctx, r.Obligations, err) + } //nolint:errorlint // callers intended to test error equality directly if _, err = io.Copy(pt, r); err != nil && err != io.EOF { return nil, formatDecryptError(ctx, r.Obligations, err) diff --git a/otdfctl/pkg/tracing/tracing.go b/otdfctl/pkg/tracing/tracing.go new file mode 100644 index 0000000000..ec2ac407f0 --- /dev/null +++ b/otdfctl/pkg/tracing/tracing.go @@ -0,0 +1,106 @@ +// Package tracing wires optional OpenTelemetry tracing into otdfctl. +// +// It is a strict no-op unless OTEL_EXPORTER_OTLP_ENDPOINT is set, so normal CLI +// usage pays no startup cost and never fails when no collector is configured. +// When enabled, it exports spans over OTLP/gRPC and continues any trace passed +// in via the standard TRACEPARENT environment variable, so a CLI invocation +// joins the trace of whatever launched it (e.g. the xtest pytest harness). +package tracing + +import ( + "context" + "os" + "strings" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" +) + +const ( + serviceName = "otdfctl" + + // shutdownTimeout bounds how long we wait to flush buffered spans on exit. + shutdownTimeout = 5 * time.Second +) + +// Init configures a global OTLP tracer provider and propagator when +// OTEL_EXPORTER_OTLP_ENDPOINT is set. It returns a shutdown function that +// flushes buffered spans (a no-op when tracing is disabled) and a bool +// reporting whether tracing was enabled. +func Init(ctx context.Context) (func(), bool) { + endpoint := os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT") + if endpoint == "" { + return func() {}, false + } + + exporter, err := otlptracegrpc.New(ctx, exporterOptions(endpoint)...) + if err != nil { + // Tracing is best-effort: never block the CLI because a collector is + // unreachable or misconfigured. + return func() {}, false + } + + res, err := resource.Merge( + resource.Default(), + resource.NewSchemaless(attribute.String("service.name", serviceName)), + ) + if err != nil { + res = resource.NewSchemaless(attribute.String("service.name", serviceName)) + } + + tp := sdktrace.NewTracerProvider( + sdktrace.WithBatcher(exporter), + sdktrace.WithResource(res), + ) + otel.SetTracerProvider(tp) + otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator( + propagation.TraceContext{}, + propagation.Baggage{}, + )) + + shutdown := func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), shutdownTimeout) + defer cancel() + _ = tp.Shutdown(shutdownCtx) + } + return shutdown, true +} + +// ExtractParentFromEnv returns a context carrying the remote span context +// described by the TRACEPARENT/TRACESTATE environment variables, so spans +// started from it become children of the caller's trace. If TRACEPARENT is +// unset the input context is returned unchanged. +func ExtractParentFromEnv(ctx context.Context) context.Context { + carrier := propagation.MapCarrier{} + if tp := os.Getenv("TRACEPARENT"); tp != "" { + carrier["traceparent"] = tp + } + if ts := os.Getenv("TRACESTATE"); ts != "" { + carrier["tracestate"] = ts + } + return otel.GetTextMapPropagator().Extract(ctx, carrier) +} + +// exporterOptions derives the OTLP/gRPC dial options from the endpoint, +// honoring an optional http:// or https:// scheme to choose (in)secure +// transport. A bare host:port defaults to insecure for local development. +func exporterOptions(endpoint string) []otlptracegrpc.Option { + insecure := true + switch { + case strings.HasPrefix(endpoint, "https://"): + insecure = false + endpoint = strings.TrimPrefix(endpoint, "https://") + case strings.HasPrefix(endpoint, "http://"): + endpoint = strings.TrimPrefix(endpoint, "http://") + } + opts := []otlptracegrpc.Option{otlptracegrpc.WithEndpoint(endpoint)} + if insecure { + opts = append(opts, otlptracegrpc.WithInsecure()) + } + return opts +} diff --git a/otdfctl/pkg/tracing/tracing_test.go b/otdfctl/pkg/tracing/tracing_test.go new file mode 100644 index 0000000000..f53637faf8 --- /dev/null +++ b/otdfctl/pkg/tracing/tracing_test.go @@ -0,0 +1,63 @@ +package tracing + +import ( + "context" + "testing" + + "go.opentelemetry.io/otel/trace" +) + +func TestInitDisabledWhenEndpointUnset(t *testing.T) { + t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "") + + shutdown, enabled := Init(context.Background()) + if enabled { + t.Fatal("expected tracing to be disabled when OTEL_EXPORTER_OTLP_ENDPOINT is unset") + } + if shutdown == nil { + t.Fatal("shutdown must be a non-nil no-op even when disabled") + } + shutdown() // must not panic +} + +func TestExtractParentFromEnv(t *testing.T) { + // A valid W3C traceparent for trace 0af7651916cd43dd8448eb211c80319c. + const traceparent = "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01" + t.Setenv("TRACEPARENT", traceparent) + + // Enable a provider so the global propagator is the W3C TraceContext. + t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "localhost:4317") + shutdown, enabled := Init(context.Background()) + if !enabled { + t.Fatal("expected tracing enabled") + } + defer shutdown() + + ctx := ExtractParentFromEnv(context.Background()) + sc := trace.SpanContextFromContext(ctx) + if !sc.IsValid() { + t.Fatal("expected a valid remote span context from TRACEPARENT") + } + if got := sc.TraceID().String(); got != "0af7651916cd43dd8448eb211c80319c" { + t.Errorf("trace id = %s, want 0af7651916cd43dd8448eb211c80319c", got) + } +} + +func TestExtractParentFromEnvNoParent(t *testing.T) { + t.Setenv("TRACEPARENT", "") + ctx := ExtractParentFromEnv(context.Background()) + if trace.SpanContextFromContext(ctx).IsValid() { + t.Error("expected no valid span context when TRACEPARENT is unset") + } +} + +func TestExporterOptionsScheme(t *testing.T) { + // Bare host:port and http:// default to insecure; https:// stays secure. + // We can't inspect the opaque options directly, so just assert the call + // returns the expected count and does not panic for each scheme. + for _, ep := range []string{"localhost:4317", "http://localhost:4317", "https://collector:4317"} { + if got := exporterOptions(ep); len(got) == 0 { + t.Errorf("exporterOptions(%q) returned no options", ep) + } + } +} diff --git a/service/logger/contextHandler.go b/service/logger/contextHandler.go index d9d2460ae4..e28949fe57 100644 --- a/service/logger/contextHandler.go +++ b/service/logger/contextHandler.go @@ -7,6 +7,7 @@ import ( "github.com/google/uuid" sdkAudit "github.com/opentdf/platform/sdk/audit" "github.com/opentdf/platform/service/logger/audit" + "go.opentelemetry.io/otel/trace" ) // ContextHandler is a custom slog.Handler that adds context attributes to log records from values set to the @@ -30,6 +31,16 @@ func (h *ContextHandler) Handle(ctx context.Context, r slog.Record) error { ) } + // Correlate logs with distributed traces: stamp trace_id/span_id from the + // active span so a log line links back to its Jaeger trace. Independent of + // RequestID so logs emitted outside the audit path still correlate. + if sc := trace.SpanContextFromContext(ctx); sc.IsValid() { + r.AddAttrs( + slog.String("trace_id", sc.TraceID().String()), + slog.String("span_id", sc.SpanID().String()), + ) + } + return h.handler.Handle(ctx, r) } diff --git a/service/logger/contextHandler_test.go b/service/logger/contextHandler_test.go new file mode 100644 index 0000000000..61b9719514 --- /dev/null +++ b/service/logger/contextHandler_test.go @@ -0,0 +1,64 @@ +package logger + +import ( + "bytes" + "context" + "encoding/json" + "log/slog" + "testing" + + "go.opentelemetry.io/otel/trace" +) + +// newHandler wires a ContextHandler around a JSON handler writing to buf. +func newHandler(buf *bytes.Buffer) slog.Handler { + return &ContextHandler{handler: slog.NewJSONHandler(buf, &slog.HandlerOptions{Level: slog.LevelDebug})} +} + +func logAndParse(t *testing.T, ctx context.Context) map[string]any { + t.Helper() + var buf bytes.Buffer + l := slog.New(newHandler(&buf)) + l.InfoContext(ctx, "test message") + + var got map[string]any + if err := json.Unmarshal(buf.Bytes(), &got); err != nil { + t.Fatalf("failed to parse log line %q: %v", buf.String(), err) + } + return got +} + +func TestContextHandler_InjectsTraceAndSpanID(t *testing.T) { + traceID, err := trace.TraceIDFromHex("0102030405060708090a0b0c0d0e0f10") + if err != nil { + t.Fatal(err) + } + spanID, err := trace.SpanIDFromHex("0102030405060708") + if err != nil { + t.Fatal(err) + } + sc := trace.NewSpanContext(trace.SpanContextConfig{ + TraceID: traceID, + SpanID: spanID, + TraceFlags: trace.FlagsSampled, + }) + ctx := trace.ContextWithSpanContext(context.Background(), sc) + + got := logAndParse(t, ctx) + if got["trace_id"] != traceID.String() { + t.Errorf("trace_id = %v, want %s", got["trace_id"], traceID.String()) + } + if got["span_id"] != spanID.String() { + t.Errorf("span_id = %v, want %s", got["span_id"], spanID.String()) + } +} + +func TestContextHandler_NoSpanNoTraceFields(t *testing.T) { + got := logAndParse(t, context.Background()) + if _, ok := got["trace_id"]; ok { + t.Errorf("trace_id should be absent without an active span, got %v", got["trace_id"]) + } + if _, ok := got["span_id"]; ok { + t.Errorf("span_id should be absent without an active span, got %v", got["span_id"]) + } +}