8000 Implement Meter attributes in both exporters by pabloos · Pull Request #11 · pabloos/flow · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions x/otel/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ module github.com/pabloos/flow/x/otel
go 1.23

require (
github.com/pabloos/flow v0.6.0
github.com/pabloos/flow v0.7.0
go.opentelemetry.io/otel v1.32.0
go.opentelemetry.io/otel/metric v1.32.0
go.opentelemetry.io/otel/sdk/metric v1.32.0
)
Expand All @@ -12,7 +13,6 @@ require (
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/google/uuid v1.6.0 // indirect
go.opentelemetry.io/otel v1.32.0 // indirect
go.opentelemetry.io/otel/sdk v1.32.0 // indirect
go.opentelemetry.io/otel/trace v1.32.0 // indirect
golang.org/x/sys v0.27.0 // indirect
Expand Down
4 changes: 2 additions & 2 deletions x/otel/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/pabloos/flow v0.6.0 h1:EtEa3QsLRDLW/6TEjgbgOlpIY8BY+Ejr/LS22voOAk0=
github.com/pabloos/flow v0.6.0/go.mod h1:8YM8kMhmio8UX5xgrejVExH8QXfQf0iIC6BVDL33IRc=
github.com/pabloos/flow v0.7.0 h1:VZ3ad5Whc9ptqqBVr4nmqZT3YsxeQ5JXmQ8I1LvCtLA=
github.com/pabloos/flow v0.7.0/go.mod h1:8YM8kMhmio8UX5xgrejVExH8QXfQf0iIC6BVDL33IRc=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
Expand Down
65 changes: 53 additions & 12 deletions x/otel/otel.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"time"

"github.com/pabloos/flow"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/metric"
)

Expand Down Expand Up @@ -39,10 +40,35 @@ type meter struct {
histograms map[string]flow.Histogram
}

func (m *meter) Counter(name string) flow.Counter {
// key folds name and attrs into one instrument identity: attributed and
// unattributed instruments of the same name are distinct series.
func key(name string, attrs []flow.Attr) string {
k := name
for _, a := range attrs {
k += "|" + a.Key + "=" + a.Value
}
return k
}

// kvs converts flow attrs to OpenTelemetry attributes. OTel attaches them per
// measurement, so the option is precomputed once here and reused on every
// Add/Record, keeping the recording path allocation-free.
func kvs(attrs []flow.Attr) []attribute.KeyValue {
if len(attrs) == 0 {
return nil
}
out := make([]attribute.KeyValue, len(attrs))
for i, a := range attrs {
out[i] = attribute.String(a.Key, a.Value)
}
return out
}

func (m *meter) Counter(name string, attrs ...flow.Attr) flow.Counter {
m.mu.Lock()
defer m.mu.Unlock()
if c, ok := m.counters[name]; ok {
k := key(name, attrs)
if c, ok := m.counters[k]; ok {
return c
}
inst, err := m.m.Int64Counter("flow."+name,
Expand All @@ -52,15 +78,19 @@ func (m *meter) Counter(name string) flow.Counter {
// condition: surface it at setup rather than silently dropping metrics.
panic("flow/x/otel: " + err.Error())
}
c := counter{inst}
m.counters[name] = c
c := counter{c: inst}
if set := kvs(attrs); len(set) > 0 {
c.opts = []metric.AddOption{metric.WithAttributes(set...)}
}
m.counters[k] = c
return c
}

func (m *meter) Histogram(name string) flow.Histogram {
func (m *meter) Histogram(name string, attrs ...flow.Attr) flow.Histogram {
m.mu.Lock()
defer m.mu.Unlock()
if h, ok := m.histograms[name]; ok {
k := key(name, attrs)
if h, ok := m.histograms[k]; ok {
return h
}
inst, err := m.m.Float64Histogram("flow."+name,
Expand All @@ -70,15 +100,26 @@ func (m *meter) Histogram(name string) flow.Histogram {
if err != nil {
panic("flow/x/otel: " + err.Error())
}
h := histogram{inst}
m.histograms[name] = h
h := histogram{h: inst}
if set := kvs(attrs); len(set) > 0 {
h.opts = []metric.RecordOption{metric.WithAttributes(set...)}
}
m.histograms[k] = h
return h
}

type counter struct{ c metric.Int64Counter }
type counter struct {
c metric.Int64Counter
opts []metric.AddOption
}

func (c counter) Add(n int64) { c.c.Add(context.Background(), n) }
func (c counter) Add(n int64) { c.c.Add(context.Background(), n, c.o 8000 pts...) }

type histogram struct{ h metric.Float64Histogram }
type histogram struct {
h metric.Float64Histogram
opts []metric.RecordOption
}

func (h histogram) Record(d time.Duration) { h.h.Record(context.Background(), d.Seconds()) }
func (h histogram) Record(d time.Duration) {
h.h.Record(context.Background(), d.Seconds(), h.opts...)
}
26 changes: 26 additions & 0 deletions x/otel/otel_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,32 @@ func TestMeterReuseAcrossRuns(t *testing.T) {
}
}

// flow.Attr becomes an OpenTelemetry attribute on every measurement, and attrs
// are part of an instrument's identity: same name, different attrs means
// separate data points.
func TestAttrsBecomeOtelAttributes(t *testing.T) {
reader := sdkmetric.NewManualReader()
provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader))
m := flowotel.NewMeter(provider.Meter("flow-test"))

m.Counter("processed", flow.Attr{Key: "worker", Value: "0"}).Add(5)
m.Counter("processed", flow.Attr{Key: "worker", Value: "1"}).Add(7)

sum, ok := findMetric(t, collect(t, reader), "flow.processed").Data.(metricdata.Sum[int64])
if !ok {
t.Fatal("flow.processed is not an Int64 Sum")
}
got := map[string]int64{}
for _, dp := range sum.DataPoints {
if v, found := dp.Attributes.Value("worker"); found {
got[v.AsString()] = dp.Value
}
}
if len(got) != 2 || got["0"] != 5 || got["1"] != 7 {
t.Fatalf("worker data points = %v, want {0:5, 1:7}", got)
}
}

func collect(t *testing.T, r sdkmetric.Reader) *metricdata.ResourceMetrics {
t.Helper()
rm := new(metricdata.ResourceMetrics)
Expand Down
2 changes: 1 addition & 1 deletion x/prometheus/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ module github.com/pabloos/flow/x/prometheus
go 1.23

require (
github.com/pabloos/flow v0.6.0
github.com/pabloos/flow v0.7.0
github.com/prometheus/client_golang v1.20.5
)

Expand Down
4 changes: 2 additions & 2 deletions x/prometheus/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/pabloos/flow v0.6.0 h1:EtEa3QsLRDLW/6TEjgbgOlpIY8BY+Ejr/LS22voOAk0=
github.com/pabloos/flow v0.6.0/go.mod h1:8YM8kMhmio8UX5xgrejVExH8QXfQf0iIC6BVDL33IRc=
github.com/pabloos/flow v0.7.0 h1:VZ3ad5Whc9ptqqBVr4nmqZT3YsxeQ5JXmQ8I1LvCtLA=
github.com/pabloos/flow v0.7.0/go.mod h1:8YM8kMhmio8UX5xgrejVExH8QXfQf0iIC6BVDL33IRc=
github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y=
github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
Expand Down
53 changes: 40 additions & 13 deletions x/prometheus/prometheus.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,34 +32,61 @@ type meter struct {
histograms map[string]flow.Histogram
}

func (m *meter) Counter(name string) flow.Counter {
// key folds name and attrs into one instrument identity: attributed and
// unattributed instruments of the same name are distinct series.
func key(name string, attrs []flow.Attr) string {
k := name
for _, a := range attrs {
k += "|" + a.Key + "=" + a.Value
}
return k
}

// labels turns flow attrs into Prometheus const labels, baked into the
// instrument at creation so recording stays allocation-free.
func labels(attrs []flow.Attr) prometheus.Labels {
if len(attrs) == 0 {
return nil
}
l := make(prometheus.Labels, len(attrs))
for _, a := range attrs {
l[a.Key] = a.Value
}
return l
}

func (m *meter) Counter(name string, attrs ...flow.Attr) flow.Counter {
m.mu.Lock()
defer m.mu.Unlock()
if c, ok := m.counters[name]; ok {
k := key(name, attrs)
if c, ok := m.counters[k]; ok {
return c
}
c := counter{m.factory.NewCounter(prometheus.CounterOpts{
Namespace: "flow",
Name: name + "_total",
Help: "flow pipeline " + name + " count",
Namespace: "flow",
Name: name + "_total",
Help: "flow pipeline " + name + " count",
ConstLabels: labels(attrs),
})}
m.counters[name] = c
m.counters[k] = c
return c
}

func (m *meter) Histogram(name string) flow.Histogram {
func (m *meter) Histogram(name string, attrs ...flow.Attr) flow.Histogram {
m.mu.Lock()
defer m.mu.Unlock()
if h, ok := m.histograms[name]; ok {
k := key(name, attrs)
if h, ok := m.histograms[k]; ok {
return h
}
h := histogram{m.factory.NewHistogram(prometheus.HistogramOpts{
Namespace: "flow",
Name: name + "_seconds",
Help: "flow pipeline " + name + " in seconds",
Buckets: prometheus.DefBuckets,
Namespace: "flow",
Name: name + "_seconds",
Help: "flow pipeline " + name + " in seconds",
Buckets: prometheus.DefBuckets,
ConstLabels: labels(attrs),
})}
m.histograms[name] = h
m.histograms[k] = h
return h
}

Expand Down
31 changes: 31 additions & 0 deletions x/prometheus/prometheus_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,37 @@ func TestMeterReuseAcrossRuns(t *testing.T) {
}
}

// flow.Attr becomes a Prometheus const label, and attrs are part of an
// instrument's identity: same name, different attrs means separate series.
func TestAttrsBecomeConstLabels(t *testing.T) {
reg := prometheus.NewRegistry()
m := flowprom.NewMeter(reg)

m.Counter("processed", flow.Attr{Key: "worker", Value: "0"}).Add(5)
m.Counter("processed", flow.Attr{Key: "worker", Value: "1"}).Add(7)

mfs, err := reg.Gather()
if err != nil {
t.Fatal(err)
}
got := map[string]float64{}
for _, mf := range mfs {
if mf.GetName() != "flow_processed_total" {
continue
}
for _, metric := range mf.GetMetric() {
for _, l := range metric.GetLabel() {
if l.GetName() == "worker" {
got[l.GetValue()] = metric.GetCounter().GetValue()
}
}
}
}
if len(got) != 2 || got["0"] != 5 || got["1"] != 7 {
t.Fatalf("worker series = %v, want {0:5, 1:7}", got)
}
}

func counterValue(t *testing.T, reg *prometheus.Registry, name string) float64 {
t.Helper()
mfs, err := reg.Gather()
Expand Down
Loading
0