diff --git a/go.mod b/go.mod index aaa6de8..f8d2090 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.23.4 require ( github.com/caarlos0/env/v9 v9.0.0 github.com/rs/zerolog v1.33.0 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 go.opentelemetry.io/otel v1.33.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.33.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 @@ -18,6 +19,11 @@ require ( gopkg.in/yaml.v3 v3.0.1 ) +require ( + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/klauspost/compress v1.17.11 // indirect +) + require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect @@ -25,22 +31,22 @@ require ( github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.25.1 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/prometheus/client_golang v1.20.5 // indirect + github.com/prometheus/client_golang v1.20.5 github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.61.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 // indirect - go.opentelemetry.io/otel/metric v1.33.0 // indirect + go.opentelemetry.io/otel/metric v1.33.0 go.opentelemetry.io/proto/otlp v1.4.0 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/text v0.21.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 // indirect - google.golang.org/grpc v1.68.1 // indirect - google.golang.org/protobuf v1.35.2 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250102185135-69823020774d // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250102185135-69823020774d // indirect + google.golang.org/grpc v1.69.2 // indirect + google.golang.org/protobuf v1.36.1 // indirect ) diff --git a/go.sum b/go.sum index 1ec9c9a..7237727 100644 --- a/go.sum +++ b/go.sum @@ -9,6 +9,8 @@ github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -21,10 +23,10 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 h1:TmHmbvxPmaegwhDubVz0lICL0J5Ka2vwTzhoePEXsGE= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0/go.mod h1:qztMSjm835F2bXf+5HKAPIS5qsmQDqZna/PgVt4rWtI= -github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= -github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.25.1 h1:VNqngBF40hVlDloBruUehVYC3ArSgIyScOAyMRqBxRg= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.25.1/go.mod h1:RBRO7fro65R6tjKzYgLAFo0t1QEXY1Dp+i/bvpRiqiQ= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -34,8 +36,9 @@ github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+ github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -58,6 +61,8 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 h1:yd02MEjBdJkG3uabWP9apV+OuWRIXGDuJEUJbOHmCFU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0/go.mod h1:umTcuxiv1n/s/S6/c2AT/g2CQ7u5C59sHDNmfSwgz7Q= go.opentelemetry.io/otel v1.33.0 h1:/FerN9bax5LoK51X/sI0SVYrjSE0/yUL7DpxW4K3FWw= go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.33.0 h1:7F29RDmnlqk6B5d+sUqemt8TBfDqxryYW5gX6L74RFA= @@ -84,8 +89,8 @@ go.opentelemetry.io/proto/otlp v1.4.0 h1:TA9WRvW6zMwP+Ssb6fLoUIuirti1gGbP28GcKG1 go.opentelemetry.io/proto/otlp v1.4.0/go.mod h1:PPBWZIP98o2ElSqI35IHfu7hIhSwvc5N38Jw8pXuGFY= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -93,14 +98,14 @@ golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 h1:CkkIfIt50+lT6NHAVoRYEyAvQGFM7xEwXUUywFvEb3Q= -google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576/go.mod h1:1R3kvZ1dtP3+4p4d3G8uJ8rFk/fWlScl38vanWACI08= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 h1:8ZmaLZE4XWrtU3MyClkYqqtl6Oegr3235h7jxsDyqCY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576/go.mod h1:5uTbfoYQed2U9p3KIj2/Zzm02PYhndfdmML0qC3q3FU= -google.golang.org/grpc v1.68.1 h1:oI5oTa11+ng8r8XMMN7jAOmWfPZWbYpCFaMUTACxkM0= -google.golang.org/grpc v1.68.1/go.mod h1:+q1XYFJjShcqn0QZHvCyeR4CXPA+llXIeUIfIe00waw= -google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io= -google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/genproto/googleapis/api v0.0.0-20250102185135-69823020774d h1:H8tOf8XM88HvKqLTxe755haY6r1fqqzLbEnfrmLXlSA= +google.golang.org/genproto/googleapis/api v0.0.0-20250102185135-69823020774d/go.mod h1:2v7Z7gP2ZUOGsaFyxATQSRoBnKygqVq2Cwnvom7QiqY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250102185135-69823020774d h1:xJJRGY7TJcvIlpSrN3K6LAWgNFUILlO+OMAqtg9aqnw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250102185135-69823020774d/go.mod h1:3ENsm/5D1mzDyhpzeRi1NR784I0BcofWBoSc5QqqMK4= +google.golang.org/grpc v1.69.2 h1:U3S9QEtbXC0bYNvRtcoklF3xGtLViumSYxWykJS+7AU= +google.golang.org/grpc v1.69.2/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= +google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= +google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/main.go b/main.go index 42d9eed..5302361 100644 --- a/main.go +++ b/main.go @@ -6,68 +6,132 @@ // endpoint. // // Configuration and logger stored in context +// Reference implementation of the provided packages package main import ( "context" + "net/http" "os" "os/signal" + "sync" "time" "github.com/rs/zerolog" - "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" "golang.org/x/sys/unix" "gitea.libretechconsulting.com/rmcguire/go-http-server-with-otel/pkg/config" "gitea.libretechconsulting.com/rmcguire/go-http-server-with-otel/pkg/logging" - "gitea.libretechconsulting.com/rmcguire/go-http-server-with-otel/pkg/observability" + "gitea.libretechconsulting.com/rmcguire/go-http-server-with-otel/pkg/otel" + "gitea.libretechconsulting.com/rmcguire/go-http-server-with-otel/pkg/srv" +) + +var ( + cfg *config.AppConfig + l *zerolog.Logger + tracer trace.Tracer ) func main() { ctx, cncl := signal.NotifyContext(context.Background(), os.Interrupt, unix.SIGTERM) defer cncl() - // Set up app config and logging - ctx, err := config.LoadConfig(ctx) - if err != nil { - panic(err) - } + shutdownFuncs := make([]func(context.Context) error, 0) - cfg := config.MustFromCtx(ctx) - ctx = logging.MustInitLogging(ctx) - l := zerolog.Ctx(ctx) + // Load configuration and setup logging + ctx = setupConfigAndLogging(ctx) // Set up OTEL - opts := make([]observability.Option, 0) - if cfg.Logging.Level == "trace" { - opts = append(opts, observability.EnableStdoutExporter) + ctx, otelShutdown := otel.Init(ctx) + shutdownFuncs = append(shutdownFuncs, otelShutdown) + tracer = otel.MustTracerFromCtx(ctx) + + // Start App + ctx, initSpan := tracer.Start(ctx, "init") + + // Start HTTP Server + dummyFuncs := []srv.HTTPFunc{ + { + Path: "/dummy", + HandlerFunc: func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("hello world")) + }, + }, } - shutdown := observability.Init(ctx, opts...) - defer func() { - shutdownCtx, cncl := context.WithTimeout(context.Background(), 30*time.Second) - defer cncl() - shutdown(shutdownCtx) - }() + httpShutdown, httpDone := srv.MustInitHTTPServer(ctx, dummyFuncs...) + shutdownFuncs = append(shutdownFuncs, httpShutdown) - // Begin Server init - tracer := otel.Tracer(cfg.Name) - _, initSpan := tracer.Start(ctx, "init") - - // TODO: HTTP Server - - // App Ready - l.Trace().Any("config", *cfg).Send() + // Startup Complete l.Info(). Str("name", cfg.Name). Str("version", cfg.Version). Str("logLevel", cfg.Logging.Level). Msg("app initialized") - initSpan.SetStatus(codes.Ok, "") initSpan.End() - <-ctx.Done() + // Wait for signal + select { + case <-httpDone: + l.Warn().Msg("shutting down early on http server done") + case <-ctx.Done(): + l.Warn().Str("reason", ctx.Err().Error()). + Msg("shutting down on context done") + } + shutdown(shutdownFuncs...) } -func init() {} +func shutdown(shutdownFuncs ...func(context.Context) error) { + now := time.Now() + + doneCtx, cncl := context.WithTimeout(context.Background(), 15*time.Second) + defer func() { + if doneCtx.Err() == context.DeadlineExceeded { + l.Err(doneCtx.Err()). + Dur("shutdownTime", time.Since(now)). + Msg("app shutdown aborted") + } else { + l.Info(). + Int("shutdownFuncsCalled", len(shutdownFuncs)). + Dur("shutdownTime", time.Since(now)). + Msg("app shutdown normally") + } + cncl() + }() + + doneCtx, span := tracer.Start(doneCtx, "shutdown") + defer span.End() + + var wg sync.WaitGroup + wg.Add(len(shutdownFuncs)) + + for _, f := range shutdownFuncs { + go func() { + defer wg.Done() + err := f(doneCtx) + if err != nil { + span.SetStatus(codes.Error, "shutdown failed") + span.RecordError(err) + l.Err(err).Send() + } + }() + } + + wg.Wait() +} + +func setupConfigAndLogging(ctx context.Context) context.Context { + ctx, err := config.LoadConfig(ctx) + if err != nil { + panic(err) + } + + cfg = config.MustFromCtx(ctx) + ctx = logging.MustInitLogging(ctx) + l = zerolog.Ctx(ctx) + + l.Trace().Any("config", *cfg).Send() + return ctx +} diff --git a/pkg/config/types.go b/pkg/config/types.go index 87cb31e..bf8d32a 100644 --- a/pkg/config/types.go +++ b/pkg/config/types.go @@ -57,12 +57,14 @@ const ( // HTTP Configuration type HTTPConfig struct { Listen string `yaml:"listen" env:"APP_HTTP_LISTEN" envDefault:"127.0.0.1:8080"` - RequestTimeout int `yaml:"request_timeout" env:"APP_HTTP_REQUEST_TIMEOUT" envDefault:"30"` + RequestTimeout int `yaml:"requestTimeout" env:"APP_HTTP_REQUEST_TIMEOUT" envDefault:"30"` } // OTEL Configuration type OTELConfig struct { - Enabled bool `yaml:"enabled" env:"APP_OTEL_ENABLED" envDefault:"true"` - PrometheusEnabled bool `yaml:"prometheus_enabled" env:"APP_OTEL_PROMETHEUS_ENABLED" envDefault:"true"` - PrometheusPath string `yaml:"prometheus_path" env:"APP_OTEL_PROMETHEUS_PATH" envDefault:"/metrics"` + Enabled bool `yaml:"enabled" env:"APP_OTEL_ENABLED" envDefault:"true"` + PrometheusEnabled bool `yaml:"prometheusEnabled" env:"APP_OTEL_PROMETHEUS_ENABLED" envDefault:"true"` + PrometheusPath string `yaml:"prometheusPath" env:"APP_OTEL_PROMETHEUS_PATH" envDefault:"/metrics"` + StdoutEnabled bool `yaml:"stdoutEnabled" env:"APP_OTEL_STDOUT_ENABLED" envDefault:"false"` + MetricIntervalSecs int `yaml:"metricIntervalSecs" env:"APP_OTEL_METRIC_INTERVAL_SECS" envDefault:"15"` } diff --git a/pkg/otel/ctx.go b/pkg/otel/ctx.go new file mode 100644 index 0000000..4597ea7 --- /dev/null +++ b/pkg/otel/ctx.go @@ -0,0 +1,54 @@ +package otel + +import ( + "context" + "errors" + + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/trace" +) + +type otelCtxKey uint8 + +const ( + ctxKeyTracer otelCtxKey = iota + ctxKeyMeter +) + +func MustTracerFromCtx(ctx context.Context) trace.Tracer { + ctxData := ctx.Value(ctxKeyTracer) + if ctxData == nil { + panic(errors.New("no tracer found in context")) + } + + tracer, ok := ctxData.(trace.Tracer) + if !ok { + panic(errors.New("invalid tracer found in context")) + } + + return tracer +} + +func AddTracerToCtx(ctx context.Context, tracer trace.Tracer) context.Context { + ctx = context.WithValue(ctx, ctxKeyTracer, tracer) + return ctx +} + +func MustMeterFromCtx(ctx context.Context) metric.Meter { + ctxData := ctx.Value(ctxKeyMeter) + if ctxData == nil { + panic(errors.New("no meter found in context")) + } + + meter, ok := ctxData.(metric.Meter) + if !ok { + panic(errors.New("invalid meter found in context")) + } + + return meter +} + +func AddMeterToCtx(ctx context.Context, meter metric.Meter) context.Context { + ctx = context.WithValue(ctx, ctxKeyMeter, meter) + return ctx +} diff --git a/pkg/observability/otel.go b/pkg/otel/otel.go similarity index 73% rename from pkg/observability/otel.go rename to pkg/otel/otel.go index 4fdaf6e..72cd18a 100644 --- a/pkg/observability/otel.go +++ b/pkg/otel/otel.go @@ -1,4 +1,4 @@ -package observability +package otel import ( "context" @@ -14,31 +14,64 @@ import ( "go.opentelemetry.io/otel/exporters/stdout/stdoutmetric" "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" + "gitea.libretechconsulting.com/rmcguire/go-http-server-with-otel/pkg/config" + + noopMetric "go.opentelemetry.io/otel/metric/noop" "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/resource" traceSDK "go.opentelemetry.io/otel/sdk/trace" semconv "go.opentelemetry.io/otel/semconv/v1.24.0" trace "go.opentelemetry.io/otel/trace" + "go.opentelemetry.io/otel/trace/noop" ) -const defaultMetricExportInterval = 15 * time.Second - +// OTEL Options var ( - EnableStdoutExporter Option = enableStdoutExporter{} + EnableStdoutExporter Option = enableStdoutExporter{} + EnablePrometheusExporter Option = enablePrometheusExporter{} // Overide the default metric export interval WithMetricExportInterval = func(interval time.Duration) Option { return exportInterval{interval: interval} } ) -func Init(ctx context.Context, options ...Option) (shutdown func(context.Context) error) { - var ( - shutdownFuncs []func(context.Context) error - s = &settings{ - MetricExportInterval: defaultMetricExportInterval, +const defMetricInterval = 15 * time.Second + +// Context must carry config.AppConfig +func Init(ctx context.Context) (context.Context, func(context.Context) error) { + cfg := config.MustFromCtx(ctx) + + // Nothing to do here if not enabled + if !cfg.OTEL.Enabled { + opentelemetry.SetMeterProvider(noopMetric.NewMeterProvider()) + opentelemetry.SetTracerProvider(noop.NewTracerProvider()) + return ctx, func(context.Context) error { + return nil } + } + + var ( + metricInterval = defMetricInterval + options = make([]Option, 0) + s = &settings{} + shutdownFuncs []func(context.Context) error ) + + // Prepare settings for OTEL from configuration + if cfg.OTEL.StdoutEnabled { + options = append(options, EnableStdoutExporter) + } + if cfg.OTEL.MetricIntervalSecs > 0 { + metricInterval = time.Duration(cfg.OTEL.MetricIntervalSecs) * time.Second + } + if cfg.OTEL.PrometheusEnabled { + options = append(options, EnablePrometheusExporter) + } + options = append(options, + WithMetricExportInterval(metricInterval)) + + // Apply settings for _, opt := range options { opt.apply(s) } @@ -46,7 +79,7 @@ func Init(ctx context.Context, options ...Option) (shutdown func(context.Context // shutdown calls cleanup functions registered via shutdownFuncs. // The errors from the calls are joined. // Each registered cleanup will be invoked once. - shutdown = func(ctx context.Context) error { + shutdown := func(ctx context.Context) error { var err error for _, fn := range shutdownFuncs { err = errors.Join(err, fn(ctx)) @@ -66,24 +99,30 @@ func Init(ctx context.Context, options ...Option) (shutdown func(context.Context meterProvider, err := s.newMeterProvider(ctx) if err != nil { handleErr(err) - return + return ctx, shutdown } shutdownFuncs = append(shutdownFuncs, meterProvider.Shutdown) opentelemetry.SetMeterProvider(meterProvider) + meter := opentelemetry.Meter(cfg.Name) + ctx = AddMeterToCtx(ctx, meter) + // Set up tracing opentelemetry.SetTextMapPropagator(newPropagator()) var tracerProvider *traceSDK.TracerProvider tracerProvider, err = s.newTracerProvider(ctx) if err != nil { handleErr(err) - return + return ctx, shutdown } shutdownFuncs = append(shutdownFuncs, tracerProvider.Shutdown) opentelemetry.SetTracerProvider(tracerProvider) - return + tracer := opentelemetry.Tracer(cfg.Name) + ctx = AddTracerToCtx(ctx, tracer) + + return ctx, shutdown } func newPropagator() propagation.TextMapPropagator { @@ -162,7 +201,7 @@ func (s *settings) newMeterProvider(ctx context.Context) (*metric.MeterProvider, metric.WithReader( metric.NewPeriodicReader( otlpExporter, - metric.WithInterval(defaultMetricExportInterval), + metric.WithInterval(s.MetricExportInterval), ), ), metric.WithResource(newResource()), diff --git a/pkg/observability/settings.go b/pkg/otel/settings.go similarity index 59% rename from pkg/observability/settings.go rename to pkg/otel/settings.go index dd01e55..3f12f2c 100644 --- a/pkg/observability/settings.go +++ b/pkg/otel/settings.go @@ -1,10 +1,11 @@ -package observability +package otel import "time" type settings struct { - EnableStdoutExporter bool - MetricExportInterval time.Duration + EnableStdoutExporter bool + EnablePrometheusExporter bool + MetricExportInterval time.Duration } type Option interface { @@ -19,6 +20,14 @@ func (setting enableStdoutExporter) apply(o *settings) { o.EnableStdoutExporter = true } +type enablePrometheusExporter struct { + Option +} + +func (setting enablePrometheusExporter) apply(o *settings) { + o.EnablePrometheusExporter = true +} + type exportInterval struct { Option interval time.Duration diff --git a/pkg/otel/util.go b/pkg/otel/util.go new file mode 100644 index 0000000..720b7f7 --- /dev/null +++ b/pkg/otel/util.go @@ -0,0 +1,30 @@ +package otel + +import ( + "context" + "strings" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/trace" + + "gitea.libretechconsulting.com/rmcguire/go-http-server-with-otel/pkg/config" +) + +func GetTracer(ctx context.Context, components ...string) trace.Tracer { + return otel.Tracer(getName(ctx, components...)) +} + +func GetMeter(ctx context.Context, components ...string) metric.Meter { + return otel.Meter(getName(ctx, components...)) +} + +func getName(ctx context.Context, components ...string) string { + cfg := config.MustFromCtx(ctx) + + path := make([]string, 0, len(components)+1) + path = append(path, cfg.Name) + path = append(path, components...) + + return strings.Join(path, ".") +} diff --git a/pkg/srv/http.go b/pkg/srv/http.go new file mode 100644 index 0000000..32681c3 --- /dev/null +++ b/pkg/srv/http.go @@ -0,0 +1,126 @@ +package srv + +import ( + "context" + "net" + "net/http" + "time" + + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/rs/zerolog" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/trace" + + "gitea.libretechconsulting.com/rmcguire/go-http-server-with-otel/pkg/config" + "gitea.libretechconsulting.com/rmcguire/go-http-server-with-otel/pkg/otel" +) + +var ( + httpMeter metric.Meter + httpTracer trace.Tracer + readTimeout = 10 * time.Second + writeTimeout = 10 * time.Second + idleTimeout = 15 * time.Second +) + +type HTTPFunc struct { + Path string + HandlerFunc http.HandlerFunc +} + +func prepHTTPServer(ctx context.Context, handleFuncs ...HTTPFunc) *http.Server { + var ( + cfg = config.MustFromCtx(ctx) + l = zerolog.Ctx(ctx) + mux = &http.ServeMux{} + ) + + // NOTE: Wraps handle func with otelhttp handler and + // inserts route tag + otelHandleFunc := func(pattern string, handlerFunc func(http.ResponseWriter, *http.Request)) { + handler := otelhttp.WithRouteTag(pattern, http.HandlerFunc(handlerFunc)) + mux.Handle(pattern, handler) // Associate pattern with handler + } + + otelHandleFunc("/health", handleHealthCheckFunc(ctx)) + otelHandleFunc("/", handleHealthCheckFunc(ctx)) + + for _, f := range handleFuncs { + otelHandleFunc(f.Path, f.HandlerFunc) + } + + // Prometheus metrics endpoint + if cfg.OTEL.PrometheusEnabled { + mux.Handle(cfg.OTEL.PrometheusPath, promhttp.Handler()) + l.Info().Str("prometheusPath", cfg.OTEL.PrometheusPath). + Msg("mounted prometheus metrics endpoint") + } + + // Add OTEL, skip health-check spans + // NOTE: Add any other span exclusions here + handler := otelhttp.NewHandler(mux, "/", + otelhttp.WithFilter(func(r *http.Request) bool { + switch r.URL.Path { + case "/health": + return false + case cfg.OTEL.PrometheusPath: + return false + default: + return true + } + })) + + return &http.Server{ + Addr: cfg.HTTP.Listen, + ReadTimeout: readTimeout, + WriteTimeout: writeTimeout, + IdleTimeout: idleTimeout, + Handler: handler, + BaseContext: func(_ net.Listener) context.Context { + return ctx + }, + } +} + +// Returns a shutdown func and a done channel if the +// server aborts abnormally. Panics on error. +func MustInitHTTPServer(ctx context.Context, funcs ...HTTPFunc) (func(context.Context) error, <-chan interface{}) { + shutdownFunc, doneChan, err := InitHTTPServer(ctx, funcs...) + if err != nil { + panic(err) + } + return shutdownFunc, doneChan +} + +// Returns a shutdown func and a done channel if the +// server aborts abnormally. Returns error on failure to start +func InitHTTPServer(ctx context.Context, funcs ...HTTPFunc) (func(context.Context) error, <-chan interface{}, error) { + l := zerolog.Ctx(ctx) + doneChan := make(chan interface{}) + + var server *http.Server + + httpMeter = otel.GetMeter(ctx, "http") + httpTracer = otel.GetTracer(ctx, "http") + + server = prepHTTPServer(ctx, funcs...) + + go func() { + l.Debug().Msg("HTTP Server Started") + err := server.ListenAndServe() + if err != nil && err != http.ErrServerClosed { + l.Err(err).Msg("HTTP server error") + } else { + l.Info().Msg("HTTP server shut down") + } + doneChan <- nil + }() + + // Shut down http server with a deadline + return func(shutdownCtx context.Context) error { + l.Debug().Msg("stopping http server") + server.Shutdown(shutdownCtx) + return nil + }, doneChan, nil +} diff --git a/pkg/srv/http_health.go b/pkg/srv/http_health.go new file mode 100644 index 0000000..a9ea37c --- /dev/null +++ b/pkg/srv/http_health.go @@ -0,0 +1,59 @@ +package srv + +import ( + "context" + "errors" + "math/rand" + "net/http" + "sync" + "time" +) + +func handleHealthCheckFunc(_ context.Context) func(w http.ResponseWriter, r *http.Request) { + // Return http handle func + return func(w http.ResponseWriter, r *http.Request) { + var err error + var healthChecksFailed bool + + // TODO: Insert useful health checks here + // For multiple checks, perform concurrently + // Consider using errors.Join() for multiple checks + var hcWg sync.WaitGroup + for range 5 { + hcWg.Add(1) + go func() { + defer hcWg.Done() + err = errors.Join(err, dummyHealthCheck(r.Context())) + }() + } + hcWg.Wait() + if err != nil { + healthChecksFailed = true + } + + // TODO: Friendly reminder... + err = errors.New("WARNING: Unimplemented health-check") + + if healthChecksFailed { + w.WriteHeader(http.StatusInternalServerError) + } + + if err != nil { + w.Write([]byte(err.Error())) + } else { + w.Write([]byte("ok")) + } + } +} + +func dummyHealthCheck(ctx context.Context) error { + workFor := rand.Intn(750) + ticker := time.NewTicker(time.Duration(time.Duration(workFor) * time.Millisecond)) + + select { + case <-ticker.C: + return nil + case <-ctx.Done(): + return ctx.Err() + } +}