OBI panicing when internal_metrics are enabled. #873
Description
I didn't only some surface level debugging but it seems like informerLag is ever being set for InternalMetricsReporter, and when it gets used in https://github.com/open-telemetry/opentelemetry-ebpf-instrumentation/blob/main/pkg/export/otel/metrics_internal.go#L293 it's crashing.
E1107 15:15:07.630276 3848521 panic.go:262] "Observed a panic" panic="runtime error: invalid memory address or nil pointer dereference" panicGoValue="\"invalid memory address or nil pointer dereference\"" stacktrace=<
goroutine 130 [running]:
k8s.io/apimachinery/pkg/util/runtime.logPanic({0x30b7800, 0xc0014eede0}, {0x2775040, 0x69df940})
/go/pkg/mod/k8s.io/[email protected]/pkg/util/runtime/runtime.go:132 +0xbc
k8s.io/apimachinery/pkg/util/runtime.handleCrash({0x30b7800, 0xc0014eedb0}, {0x2775040, 0x69df940}, {0x0, 0x0, 0xc0010c22b0?})
/go/pkg/mod/k8s.io/[email protected]/pkg/util/runtime/runtime.go:107 +0x116
k8s.io/apimachinery/pkg/util/runtime.HandleCrashWithLogger({{0x30c2e88?, 0xc00022a800?}, 0xc00071f990?}, {0x0, 0x0, 0x0})
/go/pkg/mod/k8s.io/[email protected]/pkg/util/runtime/runtime.go:91 +0x115
panic({0x2775040?, 0x69df940?})
/usr/local/go/src/runtime/panic.go:783 +0x132
go.opentelemetry.io/obi/pkg/export/otel.(*InternalMetricsReporter).InformerLag(0x482aef?, 0xc00071fb20?)
/src/pkg/export/otel/metrics_internal.go:293 +0x1c
go.opentelemetry.io/obi/pkg/kube.(*Store).On(0xc0010c2240, 0xc0023080c0)
/src/pkg/kube/store.go:224 +0x117
go.opentelemetry.io/obi/pkg/kube/kubecache/meta.(*BaseNotifier).notifyAll(0xc0007b0100, 0xc0023080c0)
/src/pkg/kube/kubecache/meta/observer.go:63 +0x174
go.opentelemetry.io/obi/pkg/kube/kubecache/meta.(*BaseNotifier).Notify(0xc0007b0100, 0x30b77c8?)
/src/pkg/kube/kubecache/meta/observer.go:49 +0x28
go.opentelemetry.io/obi/pkg/kube/kubecache/meta.(*Informers).ipInfoEventHandler.func2({0x2c118a0, 0xc002f87b30}, {0x2c118a0, 0xc00154cd20})
/src/pkg/kube/kubecache/meta/informers_init.go:572 +0x30c
k8s.io/client-go/tools/cache.ResourceEventHandlerFuncs.OnUpdate(...)
/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:266
k8s.io/client-go/tools/cache.(*processorListener).run.func1(0xc00071ff5f, 0xc00091ae60, {0x290dc00?, 0xc0028a2400?})
/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:1074 +0x149
k8s.io/client-go/tools/cache.(*processorListener).run(0xc00091ae60)
/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:1086 +0x3c
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:72 +0x4c
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start in goroutine 116
/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:70 +0x73
>
panic: runtime error: invalid memory address or nil pointer dereference [recovered, repanicked]
[signal SIGSEGV: segmentation violation code=0x1 addr=0x18 pc=0x21861bc]
goroutine 130 [running]:
k8s.io/apimachinery/pkg/util/runtime.handleCrash({0x30b7800, 0xc0014eedb0}, {0x2775040, 0x69df940}, {0x0, 0x0, 0xc0010c22b0?})
/go/pkg/mod/k8s.io/[email protected]/pkg/util/runtime/runtime.go:114 +0x1a9
k8s.io/apimachinery/pkg/util/runtime.HandleCrashWithLogger({{0x30c2e88?, 0xc00022a800?}, 0xc00071f990?}, {0x0, 0x0, 0x0})
/go/pkg/mod/k8s.io/[email protected]/pkg/util/runtime/runtime.go:91 +0x115
panic({0x2775040?, 0x69df940?})
/usr/local/go/src/runtime/panic.go:783 +0x132
go.opentelemetry.io/obi/pkg/export/otel.(*InternalMetricsReporter).InformerLag(0x482aef?, 0xc00071fb20?)
/src/pkg/export/otel/metrics_internal.go:293 +0x1c
go.opentelemetry.io/obi/pkg/kube.(*Store).On(0xc0010c2240, 0xc0023080c0)
/src/pkg/kube/store.go:224 +0x117
go.opentelemetry.io/obi/pkg/kube/kubecache/meta.(*BaseNotifier).notifyAll(0xc0007b0100, 0xc0023080c0)
/src/pkg/kube/kubecache/meta/observer.go:63 +0x174
go.opentelemetry.io/obi/pkg/kube/kubecache/meta.(*BaseNotifier).Notify(0xc0007b0100, 0x30b77c8?)
/src/pkg/kube/kubecache/meta/observer.go:49 +0x28
go.opentelemetry.io/obi/pkg/kube/kubecache/meta.(*Informers).ipInfoEventHandler.func2({0x2c118a0, 0xc002f87b30}, {0x2c118a0, 0xc00154cd20})
/src/pkg/kube/kubecache/meta/informers_init.go:572 +0x30c
k8s.io/client-go/tools/cache.ResourceEventHandlerFuncs.OnUpdate(...)
/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:266
k8s.io/client-go/tools/cache.(*processorListener).run.func1(0xc00071ff5f, 0xc00091ae60, {0x290dc00?, 0xc0028a2400?})
/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:1074 +0x149
k8s.io/client-go/tools/cache.(*processorListener).run(0xc00091ae60)
/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:1086 +0x3c
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:72 +0x4c
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start in goroutine 116
/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:70 +0x73