TODO.md

@@ -2,7 +2,6 @@
 - [x] Redis recorder panic
 
 # TODO
-- [ ] Make lightning distance unit configurable (kph|mph)
 - [ ] Set TTL on redis key
 - [ ] Add json schema to CI and README
 - [ ] Update README

api/v1alpha1/weather/weather.pb.go

@@ -298,7 +298,6 @@ type WeatherUpdate struct {
 	UpdateTimestamp     *timestamppb.Timestamp `protobuf:"bytes,28,opt,name=update_timestamp,json=updateTimestamp,proto3" json:"update_timestamp,omitempty"`
 	LightningDay        *int32                 `protobuf:"varint,29,opt,name=lightning_day,json=lightningDay,proto3,oneof" json:"lightning_day,omitempty"`
 	LightningDistance   *int32                 `protobuf:"varint,30,opt,name=lightning_distance,json=lightningDistance,proto3,oneof" json:"lightning_distance,omitempty"`
-	LightningLastTime   *timestamppb.Timestamp `protobuf:"bytes,31,opt,name=lightning_last_time,json=lightningLastTime,proto3,oneof" json:"lightning_last_time,omitempty"`
 	unknownFields       protoimpl.UnknownFields
 	sizeCache           protoimpl.SizeCache
 }
@@ -543,13 +542,6 @@ func (x *WeatherUpdate) GetLightningDistance() int32 {
 	return 0
 }
 
-func (x *WeatherUpdate) GetLightningLastTime() *timestamppb.Timestamp {
-	if x != nil {
-		return x.LightningLastTime
-	}
-	return nil
-}
-
 // Represents a temperature and humidity sensor
 type TempHumiditySensor struct {
 	state         protoimpl.MessageState `protogen:"open.v1"`
@@ -684,7 +676,7 @@ const file_weather_weather_proto_rawDesc = "" +
 	"\fstation_name\x18\x01 \x01(\tH\x00R\vstationName\x88\x01\x01\x12&\n" +
 	"\fstation_type\x18\x02 \x01(\tH\x01R\vstationType\x88\x01\x01B\x0f\n" +
 	"\r_station_nameB\x0f\n" +
-	"\r_station_type\"\xee\x0e\n" +
+	"\r_station_type\"\x85\x0e\n" +
 	"\rWeatherUpdate\x12!\n" +
 	"\fstation_name\x18\x01 \x01(\tR\vstationName\x12!\n" +
 	"\fstation_type\x18\x02 \x01(\tR\vstationType\x12\x1d\n" +
@@ -719,8 +711,7 @@ const file_weather_weather_proto_rawDesc = "" +
 	"\x15temp_humidity_sensors\x18\x1b \x03(\v2#.ambient.weather.TempHumiditySensorR\x13tempHumiditySensors\x12E\n" +
 	"\x10update_timestamp\x18\x1c \x01(\v2\x1a.google.protobuf.TimestampR\x0fupdateTimestamp\x12(\n" +
 	"\rlightning_day\x18\x1d \x01(\x05H\x16R\flightningDay\x88\x01\x01\x122\n" +
-	"\x12lightning_distance\x18\x1e \x01(\x05H\x17R\x11lightningDistance\x88\x01\x01\x12O\n" +
+	"\x12lightning_distance\x18\x1e \x01(\x05H\x17R\x11lightningDistance\x88\x01\x01B\x11\n" +
-	"\x13lightning_last_time\x18\x1f \x01(\v2\x1a.google.protobuf.TimestampH\x18R\x11lightningLastTime\x88\x01\x01B\x11\n" +
 	"\x0f_temp_outdoor_fB\x10\n" +
 	"\x0e_temp_indoor_fB\x13\n" +
 	"\x11_humidity_outdoorB\x12\n" +
@@ -744,8 +735,7 @@ const file_weather_weather_proto_rawDesc = "" +
 	"\f_dew_point_fB\x0f\n" +
 	"\r_wind_chill_fB\x10\n" +
 	"\x0e_lightning_dayB\x15\n" +
-	"\x13_lightning_distanceB\x16\n" +
+	"\x13_lightning_distance\"}\n" +
-	"\x14_lightning_last_time\"}\n" +
 	"\x12TempHumiditySensor\x12\x12\n" +
 	"\x04name\x18\x01 \x01(\tR\x04name\x12\x1a\n" +
 	"\x06temp_f\x18\x02 \x01(\x01H\x00R\x05tempF\x88\x01\x01\x12\x1f\n" +
@@ -789,12 +779,11 @@ var file_weather_weather_proto_depIdxs = []int32{
 	7, // 4: ambient.weather.WeatherUpdate.batteries:type_name -> ambient.weather.BatteryStatus
 	6, // 5: ambient.weather.WeatherUpdate.temp_humidity_sensors:type_name -> ambient.weather.TempHumiditySensor
 	8, // 6: ambient.weather.WeatherUpdate.update_timestamp:type_name -> google.protobuf.Timestamp
-	8, // 7: ambient.weather.WeatherUpdate.lightning_last_time:type_name -> google.protobuf.Timestamp
+	7, // [7:7] is the sub-list for method output_type
-	8, // [8:8] is the sub-list for method output_type
+	7, // [7:7] is the sub-list for method input_type
-	8, // [8:8] is the sub-list for method input_type
+	7, // [7:7] is the sub-list for extension type_name
-	8, // [8:8] is the sub-list for extension type_name
+	7, // [7:7] is the sub-list for extension extendee
-	8, // [8:8] is the sub-list for extension extendee
+	0, // [0:7] is the sub-list for field type_name
-	0, // [0:8] is the sub-list for field type_name
 }
 
 func init() { file_weather_weather_proto_init() }

api/v1alpha1/weather/weather_service.swagger.json

@@ -316,10 +316,6 @@
         "lightningDistance": {
           "type": "integer",
           "format": "int32"
-        },
-        "lightningLastTime": {
-          "type": "string",
-          "format": "date-time"
         }
       }
     }

contrib/schema.json

@@ -121,10 +121,6 @@
           "default": "127.0.0.1",
           "type": "string"
         },
-        "redisKeyTTL": {
-          "default": "24h",
-          "type": "string"
-        },
         "redisPassword": {
           "type": "string"
         },

helm/ambient-local-exporter/Chart.yaml

@@ -15,13 +15,13 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.12
+version: 0.1.8
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "v0.18.1"
+appVersion: "v0.14.0"
 
 dependencies:
   - name: hull

pkg/ambient/ambientgrpc/mapupdate.go

@@ -61,7 +61,6 @@ func UpdateToPbUpdate(u *weather.WeatherUpdate) *pb.WeatherUpdate {
 		UpdateTimestamp:     timestamppb.New(*u.DateUTC),
 		LightningDay:        util.Int32ptr(u.LightningDay),
 		LightningDistance:   util.Int32ptr(u.LightningDistance),
-		LightningLastTime:   util.TimestampFromIntPtr(u.LightningLastTime),
 	}
 }
 

pkg/ambient/ambientgrpc/mapupdate_test.go

@@ -52,8 +52,10 @@ var mockUpdate = &weather.WeatherUpdate{
 	TempHumiditySensors: []*weather.TempHumiditySensor{
 		{Name: "sensor1", TempF: ptr.To(99.999), Humidity: nil},
 	},
-	LightningDay:      ptr.To(1),
+	LightningData: &weather.LightningData{
-	LightningDistance: ptr.To(20),
+		LightningDay:      ptr.To(1),
+		LightningDistance: ptr.To(20),
+	},
 }
 
 func TestUpdateToPbUpdate(t *testing.T) {

pkg/ambient/config/config_recorder.go

@@ -22,5 +22,4 @@ type RedisConfig struct {
 	RedisDB          int    `yaml:"redisDB" env:"REDIS_DB" default:"0" json:"redisDB,omitempty"`
 	RedisTLS         bool   `yaml:"redisTLS" env:"REDIS_TLS" default:"false" json:"redisTLS,omitempty"`
 	RedisTLSInsecure bool   `yaml:"redisTLSInsecure" env:"REDIS_TLS_INSECURE" default:"false" json:"redisTLSInsecure,omitempty"`
-	RedisKeyTTL      string `yaml:"redisKeyTTL" env:"REDIS_KEY_TTL" default:"24h" json:"redisKeyTTL,omitempty"`
 }

pkg/provider/awn/provider.go

@@ -143,9 +143,11 @@ func MapAwnUpdate(awnUpdate *AmbientWeatherUpdate) *weather.WeatherUpdate {
 			{Name: THSensor + "7", TempF: awnUpdate.Temp7F, Humidity: awnUpdate.Humidity7},
 			{Name: THSensor + "8", TempF: awnUpdate.Temp8F, Humidity: awnUpdate.Humidity8},
 		},
-		LightningDay:      awnUpdate.LightningDay,
+		LightningData: &weather.LightningData{
-		LightningDistance: awnUpdate.LightningDistance,
+			LightningDay:      awnUpdate.LightningDay,
-		LightningLastTime: awnUpdate.LightningLastTime,
+			LightningDistance: awnUpdate.LightningDistance,
+			LightningTime:     awnUpdate.LightningTime,
+		},
 	}
 }
 

pkg/provider/awn/types.go

@@ -36,7 +36,7 @@ type AmbientLightningData struct {
 	BattLightning     *int `json:"batt_lightning,omitempty" schema:"batt_lightning"`
 	LightningDay      *int `json:"lightning_day,omitempty" schema:"lightning_day" description:"count of lightning detections"`
 	LightningDistance *int `json:"lightning_distance,omitempty" schema:"lightning_distance" description:"distance in kilometers"`
-	LightningLastTime *int `json:"lightning_time,omitempty" schema:"lightning_time" description:"last lightning detection time"`
+	LightningTime     *int `json:"lightningTime,omitempty" schema:"lightningTime" description:"last lightning detection time"`
 }
 
 type AmbientTempHumiditySensors struct {

pkg/util/util.go

@@ -5,10 +5,8 @@ import (
 	"encoding/hex"
 	"fmt"
 	"slices"
-	"time"
 
 	"gitea.libretechconsulting.com/rmcguire/go-app/pkg/config"
-	"google.golang.org/protobuf/types/known/timestamppb"
 	"k8s.io/utils/ptr"
 
 	pb "gitea.libretechconsulting.com/rmcguire/ambient-local-exporter/api/v1alpha1/weather"
@@ -31,13 +29,6 @@ func Int32ptr(i *int) *int32 {
 	return ptr.To(int32(*i))
 }
 
-func TimestampFromIntPtr(i *int) *timestamppb.Timestamp {
-	if i == nil {
-		return nil
-	}
-	return timestamppb.New(time.Unix(int64(*i), 0))
-}
-
 // Generates a hash that will be consistent
 // across all running replicas
 func GetAppHash(conf *config.AppConfig) string {

pkg/weather/metrics.go

@@ -40,7 +40,6 @@ type WeatherMetrics struct {
 	// Lightning Sensor
 	LightningCountDay     metric.Int64Gauge
 	LightningLastDistance metric.Int64Gauge
-	LightningLastTime     metric.Int64Gauge
 
 	// Temp and Humidity Sensors
 	SensorTempF    metric.Float64Gauge
@@ -80,9 +79,9 @@ func (wm *WeatherMetrics) Update(u *WeatherUpdate) {
 	wm.recorder.Record(&RecordOpts{Float64Gauge: wm.DewPointF, FloatVal: u.DewPointF, Field: FieldDewPointF, Attributes: attributes, Station: u.StationConfig})
 	wm.recorder.Record(&RecordOpts{Float64Gauge: wm.WindChillF, FloatVal: u.WindChillF, Field: FieldWindChillF, Attributes: attributes, Station: u.StationConfig})
 
-	wm.recorder.Record(&RecordOpts{Int64Gauge: wm.LightningLastDistance, IntVal: u.LightningDistance, Field: FieldLightningDistance, Attributes: attributes, Station: u.StationConfig})
+	lightningAttributes := append(attributes, attribute.String("unit", "km"))
+	wm.recorder.Record(&RecordOpts{Int64Gauge: wm.LightningLastDistance, IntVal: u.LightningDistance, Field: FieldLightningDistance, Attributes: lightningAttributes, Station: u.StationConfig})
 	wm.recorder.Record(&RecordOpts{Int64Gauge: wm.LightningCountDay, IntVal: u.LightningDay, Field: FieldLightningDay, Attributes: attributes, Station: u.StationConfig})
-	wm.recorder.Record(&RecordOpts{Int64Gauge: wm.LightningLastTime, IntVal: u.LightningLastTime, Field: FieldLightningLastTime, Attributes: attributes, Station: u.StationConfig})
 
 	wm.RecordBatteries(u, attributes)
 	wm.RecordTempHumiditySensors(u, attributes)

pkg/weather/metrics_init.go

@@ -70,11 +70,7 @@ func MustInitMetrics(appCtx context.Context) *WeatherMetrics {
 	wm.LightningCountDay, _ = wm.meter.Int64Gauge(MetricPrefix+"_lightning_day",
 		metric.WithDescription("Count of lightning strikes for current day"))
 	wm.LightningLastDistance, _ = wm.meter.Int64Gauge(MetricPrefix+"_lightning_last_distance",
-		metric.WithDescription("Last measured lightning distance"),
+		metric.WithDescription("Last measured lightning distance"))
-		metric.WithUnit("km"))
-	wm.LightningLastTime, _ = wm.meter.Int64Gauge(MetricPrefix+"_lightning_last_time",
-		metric.WithDescription("Last lightning time (epoch)"),
-		metric.WithUnit("s"))
 
 	// Temp and Humidity Sensors
 	wm.SensorTempF, _ = wm.meter.Float64Gauge(MetricPrefix+"_sensor_temp_f",

pkg/weather/metrics_record.go

@@ -3,7 +3,6 @@ package weather
 import (
 	"context"
 	"errors"
-	"slices"
 
 	"github.com/rs/zerolog"
 	"go.opentelemetry.io/otel/attribute"
@@ -20,7 +19,6 @@ type MetricRecorder struct {
 type RecordOpts struct {
 	Float64Gauge metric.Float64Gauge
 	Int64Gauge   metric.Int64Gauge
-	Int64Counter metric.Int64Counter
 	IntVal       *int
 	FloatVal     *float64
 	Attributes   []attribute.KeyValue
@@ -60,39 +58,26 @@ func (r *MetricRecorder) Record(opts *RecordOpts) {
 			return
 		}
 		r.recordFloat(opts.Float64Gauge, *opts.FloatVal, opts.Attributes...)
-	} else if opts.Int64Counter != nil {
-		if opts.IntVal == nil {
-			log := r.l.Trace().Str("field", string(opts.Field))
-			if opts.Station != nil {
-				log = log.Str("station", opts.Station.Name)
-			}
-			log.Msg("Dropping nil float metric")
-			return
-		}
-		r.recordCounter(opts.Int64Counter, *opts.IntVal, opts.Attributes...)
 	}
 }
 
 func (o *RecordOpts) keep() bool {
 	// If keep fields are given, only check keep fields
 	if len(o.Station.KeepMetrics) > 0 {
-		return slices.Contains(o.Station.KeepMetrics, o.Field)
+		for _, f := range o.Station.KeepMetrics {
+			if f == o.Field {
+				return true
+			}
+		}
+		return false
 	}
 
-	return !slices.Contains(o.Station.DropMetrics, o.Field)
+	for _, f := range o.Station.DropMetrics {
-}
+		if f == o.Field {
-
+			return false
-func (r *MetricRecorder) recordCounter(
+		}
-	m metric.Int64Counter, value int, attributes ...attribute.KeyValue,
-) {
-	// Prepare metric attributes
-	options := make([]metric.AddOption, 0, len(attributes))
-	if len(attributes) > 0 {
-		options = append(options, metric.WithAttributes(attributes...))
 	}
-
+	return true
-	val := int64(value)
-	m.Add(r.ctx, val, options...)
 }
 
 func (r *MetricRecorder) recordInt(

pkg/weather/recorder/recorders/redis/redis.go

@@ -5,7 +5,6 @@ import (
 	"crypto/tls"
 	"fmt"
 	"sync"
-	"time"
 
 	"gitea.libretechconsulting.com/rmcguire/go-app/pkg/otel"
 	"go.opentelemetry.io/otel/attribute"
@@ -21,7 +20,6 @@ import (
 )
 
 const (
-	DEF_TTL     = "24h"
 	DEF_RETAIN  = 120
 	UPDATES_KEY = "weatherUpdates"
 	NAME        = "redis recorder"
@@ -29,7 +27,6 @@ const (
 
 type RedisRecorder struct {
 	baseCtx context.Context
-	keyTTL  time.Duration
 	tracer  trace.Tracer
 	redis   *redis.Client
 	config  *config.AmbientLocalExporterConfig
@@ -45,11 +42,8 @@ func (r *RedisRecorder) Ping(ctx context.Context) error {
 }
 
 func (r *RedisRecorder) Init(ctx context.Context, opts *recorders.RecorderOpts) {
-	r.baseCtx = opts.BaseCtx
-	r.log = zerolog.Ctx(r.baseCtx)
-
 	if opts.AppConfig.RecorderConfig.RedisConfig == nil {
-		r.log.Panic().Msg("refusing to init redis recorder with no redisConfig")
+		panic("refusing to init redis recorder with no redisConfig")
 	}
 
 	if opts.RetainLast < 1 {
@@ -59,18 +53,8 @@ func (r *RedisRecorder) Init(ctx context.Context, opts *recorders.RecorderOpts)
 	r.config = opts.AppConfig
 	r.keep = opts.RetainLast
 	r.RWMutex = &sync.RWMutex{}
-
+	r.baseCtx = opts.BaseCtx
-	keyTTL := opts.AppConfig.RecorderConfig.RedisConfig.RedisKeyTTL
+	r.log = zerolog.Ctx(r.baseCtx)
-	if keyTTL == "" {
-		keyTTL = DEF_TTL
-	}
-
-	var err error
-	r.keyTTL, err = time.ParseDuration(keyTTL)
-	if err != nil {
-		r.log.Panic().Str("redisKeyTTL", keyTTL).Err(err).
-			Msg("invalid redis key ttl provided")
-	}
 
 	r.tracer = otel.GetTracer(r.baseCtx, "redisRecorder")
 	ctx, span := r.tracer.Start(ctx, "redisRecorder.init", trace.WithAttributes(
@@ -78,7 +62,6 @@ func (r *RedisRecorder) Init(ctx context.Context, opts *recorders.RecorderOpts)
 		attribute.Int("retainLast", opts.RetainLast),
 		attribute.Int("redisPort", opts.AppConfig.RecorderConfig.RedisConfig.RedisPort),
 		attribute.Bool("tls", opts.AppConfig.RecorderConfig.RedisConfig.RedisTLS),
-		attribute.String("keyTTL", r.keyTTL.String()),
 	))
 	defer span.End()
 

pkg/weather/recorder/recorders/redis/set.go

@@ -48,7 +48,6 @@ func (r *RedisRecorder) set(ctx context.Context, data []byte) error {
 		pipe.LPush(ctx, r.Key(), data)
 		pipe.LTrim(ctx, r.Key(), 0, int64(r.keep)-1)
 		count = pipe.LLen(ctx, r.Key())
-		pipe.Expire(ctx, r.Key(), r.keyTTL)
 		return nil
 	})
 	if err != nil {

pkg/weather/types.go

@@ -9,52 +9,56 @@ import (
 // Stable intermediate struct containing superset of fields
 // between AWN and Wunderground style updates from Ambient devices
 type WeatherUpdate struct {
-	DateUTC         *time.Time             `json:"dateUTC,omitempty"`
+	DateUTC         *time.Time
-	StationConfig   *config.WeatherStation `json:"stationConfig,omitempty"`
+	StationConfig   *config.WeatherStation
-	StationID       *string                `json:"stationID,omitempty"`
+	StationID       *string
-	StationType     *string                `json:"stationType,omitempty"`
+	StationType     *string
-	TempOutdoorF    *float64               `json:"tempOutdoorF,omitempty"`
+	TempOutdoorF    *float64
-	TempIndoorF     *float64               `json:"tempIndoorF,omitempty"`
+	TempIndoorF     *float64
-	HumidityOudoor  *int                   `json:"humidityOudoor,omitempty"`
+	HumidityOudoor  *int
-	HumidityIndoor  *int                   `json:"humidityIndoor,omitempty"`
+	HumidityIndoor  *int
-	WindSpeedMPH    *float64               `json:"windSpeedMPH,omitempty"`
+	WindSpeedMPH    *float64
-	WindGustMPH     *float64               `json:"windGustMPH,omitempty"`
+	WindGustMPH     *float64
-	MaxDailyGust    *float64               `json:"maxDailyGust,omitempty"`
+	MaxDailyGust    *float64
-	WindDir         *int                   `json:"windDir,omitempty"`
+	WindDir         *int
-	WindDirAvg10m   *int                   `json:"windDirAvg10M,omitempty"`
+	WindDirAvg10m   *int
-	UV              *int                   `json:"uv,omitempty"`
+	UV              *int
-	SolarRadiation  *float64               `json:"solarRadiation,omitempty"`
+	SolarRadiation  *float64
-	HourlyRainIn    *float64               `json:"hourlyRainIn,omitempty"`
+	HourlyRainIn    *float64
-	EventRainIn     *float64               `json:"eventRainIn,omitempty"`
+	EventRainIn     *float64
-	DailyRainIn     *float64               `json:"dailyRainIn,omitempty"`
+	DailyRainIn     *float64
-	WeeklyRainIn    *float64               `json:"weeklyRainIn,omitempty"`
+	WeeklyRainIn    *float64
-	MonthlyRainIn   *float64               `json:"monthlyRainIn,omitempty"`
+	MonthlyRainIn   *float64
-	YearlyRainIn    *float64               `json:"yearlyRainIn,omitempty"`
+	YearlyRainIn    *float64
-	TotalRainIn     *float64               `json:"totalRainIn,omitempty"`
+	TotalRainIn     *float64
-	Batteries       []BatteryStatus        `json:"batteries,omitempty"`
+	Batteries       []BatteryStatus
-	BaromRelativeIn *float64               `json:"baromRelativeIn,omitempty"`
+	BaromRelativeIn *float64
-	BaromAbsoluteIn *float64               `json:"baromAbsoluteIn,omitempty"`
+	BaromAbsoluteIn *float64
 	// These fields may be calculated
 	// if not otherwise set
-	DewPointF  *float64 `json:"dewPointF,omitempty"`
+	DewPointF  *float64
-	WindChillF *float64 `json:"windChillF,omitempty"`
+	WindChillF *float64
 	// Extra Temp+Humidity Sensors
-	TempHumiditySensors []*TempHumiditySensor `json:"tempHumiditySensors,omitempty"`
+	TempHumiditySensors []*TempHumiditySensor
 	// Lightning Data
-	LightningDay      *int `json:"lightningDay,omitempty"`      // Count of detections
+	*LightningData
-	LightningDistance *int `json:"lightningDistance,omitempty"` // Last detection distance [kilometers]
+}
-	LightningLastTime *int `json:"lightningLastTime,omitempty"` // Last detection time
+
+type LightningData struct {
+	LightningDay      *int // Count of detections
+	LightningDistance *int // Last detection distance [kilometers]
+	LightningTime     *int // Last detection time
 }
 
 type TempHumiditySensor struct {
-	Name     string   `json:"name,omitempty"`
+	Name     string
-	TempF    *float64 `json:"tempF,omitempty"`
+	TempF    *float64
-	Humidity *int     `json:"humidity,omitempty"`
+	Humidity *int
 }
 
 type BatteryStatus struct {
-	Component string `json:"component,omitempty"`
+	Component string
-	Status    *int   `json:"status,omitempty"`
+	Status    *int
 }
 
 // CHORE: Maintain this, used to check against
@@ -90,7 +94,6 @@ const (
 	FieldSensorHumidity    = "SensorHumidity"
 	FieldLightningDay      = "LightningDay"
 	FieldLightningDistance = "LightningDistance"
-	FieldLightningLastTime = "LightningLastTime"
 )
 
 func (u *WeatherUpdate) GetStationName() string {

proto/weather/weather.proto

@@ -59,7 +59,6 @@ message WeatherUpdate {
   google.protobuf.Timestamp update_timestamp = 28;
   optional int32 lightning_day = 29;
   optional int32 lightning_distance = 30;
-  optional google.protobuf.Timestamp lightning_last_time = 31;
 }
 
 // Represents a temperature and humidity sensor