From ee2efdafb06df9380783da86868fe99b2bbcb901 Mon Sep 17 00:00:00 2001 From: Aaron Chan Date: Fri, 3 Apr 2026 11:22:15 -0400 Subject: [PATCH 1/3] Update docker files --- cmd/Containerfile | 6 ++++-- compose-services.yaml | 45 ++++++++++++++++++++++++++++++++++--------- compose.yaml | 7 +++++-- 3 files changed, 45 insertions(+), 13 deletions(-) diff --git a/cmd/Containerfile b/cmd/Containerfile index 81758ee..b7c92be 100644 --- a/cmd/Containerfile +++ b/cmd/Containerfile @@ -43,8 +43,10 @@ USER appuser ENV GSW_LOGGER_OUTPUT_PATHS=stdout -ENV GSW_DATABASE_HOST_NAME=influxdb -ENV GSW_DATABASE_PORT_NUMBER=8089 +ENV GSW_DATABASE_V2_URL=http://influxdb:8086 +ENV GSW_DATABASE_V2_TOKEN=gsw-local-dev-token +ENV GSW_DATABASE_V2_ORG=gsw +ENV GSW_DATABASE_V2_BUCKET=gsw ENV GSW_LOGGING_CONFIG=data/config/logger.yaml ENV GSW_TELEMETRY_CONFIG=data/config/backplane.yaml diff --git a/compose-services.yaml b/compose-services.yaml index e36b1f4..4f53aba 100644 --- a/compose-services.yaml +++ b/compose-services.yaml @@ -6,26 +6,52 @@ services: ports: - "127.0.0.1:3000:3000/tcp" depends_on: - influxdb: - condition: service_started + influxdb-init: + condition: service_completed_successfully environment: - GSW_GRAFANA_INFLUXDB_URL=http://influxdb:8086 - - GSW_GRAFANA_INFLUXDB_USER=root - - GSW_GRAFANA_INFLUXDB_PASSWORD=root + - GSW_GRAFANA_INFLUXDB_DB_NAME=${GSW_INFLUXDB_BUCKET:-gsw} + - GSW_GRAFANA_INFLUXDB_TOKEN=${GSW_INFLUXDB_ADMIN_TOKEN:-gsw-local-dev-token} - GF_AUTH_ANONYMOUS_ENABLED=true - GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer - GF_AUTH_BASIC_ENABLED=false volumes: - grafana-data:/var/lib/grafana influxdb: - image: influxdb:1.11-alpine + image: influxdb:2.7-alpine volumes: - - influxdb-data:/var/lib/influxdb + - influxdb-data:/var/lib/influxdb2 + - influxdb-config:/etc/influxdb2 environment: - - INFLUXDB_UDP_ENABLED=true - - INFLUXDB_UDP_DATABASE=gsw + - DOCKER_INFLUXDB_INIT_MODE=setup + - DOCKER_INFLUXDB_INIT_USERNAME=${GSW_INFLUXDB_USERNAME:-gsw} + - DOCKER_INFLUXDB_INIT_PASSWORD=${GSW_INFLUXDB_PASSWORD:-gsw-password} + - DOCKER_INFLUXDB_INIT_ORG=${GSW_INFLUXDB_ORG:-gsw} + - DOCKER_INFLUXDB_INIT_BUCKET=${GSW_INFLUXDB_BUCKET:-gsw} + - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=${GSW_INFLUXDB_ADMIN_TOKEN:-gsw-local-dev-token} ports: - - "127.0.0.1:8089:8089/udp" + - "127.0.0.1:8086:8086/tcp" + healthcheck: + test: ["CMD", "influx", "ping", "--host", "http://localhost:8086"] + interval: 5s + timeout: 5s + retries: 20 + start_period: 10s + influxdb-init: + image: influxdb:2.7-alpine + depends_on: + influxdb: + condition: service_healthy + environment: + - INFLUX_HOST=http://influxdb:8086 + - DOCKER_INFLUXDB_INIT_ORG=${GSW_INFLUXDB_ORG:-gsw} + - DOCKER_INFLUXDB_INIT_BUCKET=${GSW_INFLUXDB_BUCKET:-gsw} + - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=${GSW_INFLUXDB_ADMIN_TOKEN:-gsw-local-dev-token} + - GSW_INFLUXDB_V1_DATABASE=${GSW_INFLUXDB_BUCKET:-gsw} + - GSW_INFLUXDB_V1_RETENTION_POLICY=autogen + volumes: + - ./data/influxdb/init-v1-compat.sh:/docker-entrypoint-initdb.d/init-v1-compat.sh:ro + entrypoint: ["/bin/sh", "/docker-entrypoint-initdb.d/init-v1-compat.sh"] mosquitto: build: @@ -38,3 +64,4 @@ services: volumes: grafana-data: influxdb-data: + influxdb-config: diff --git a/compose.yaml b/compose.yaml index 0ac33c9..9b0b5e9 100644 --- a/compose.yaml +++ b/compose.yaml @@ -8,11 +8,14 @@ services: dockerfile: cmd/Containerfile depends_on: influxdb: - condition: service_started + condition: service_healthy grafana: condition: service_started environment: - - GSW_DATABASE_HOST_NAME=influxdb + - GSW_DATABASE_V2_URL=http://influxdb:8086 + - GSW_DATABASE_V2_TOKEN=${GSW_INFLUXDB_ADMIN_TOKEN:-gsw-local-dev-token} + - GSW_DATABASE_V2_ORG=${GSW_INFLUXDB_ORG:-gsw} + - GSW_DATABASE_V2_BUCKET=${GSW_INFLUXDB_BUCKET:-gsw} - GSW_TELEMETRY_CONFIG=data/config/backplane.yaml ports: - 11020:11020/udp From b6e38f30867b6a4e25b67daca84ccb27b8a5fba2 Mon Sep 17 00:00:00 2001 From: Aaron Chan Date: Fri, 3 Apr 2026 11:23:06 -0400 Subject: [PATCH 2/3] Update example yaml and the db provider yaml --- data/config/gsw_service.yaml.example | 4 +- .../datasources/gsw-influxdb-provider.yaml | 10 ++-- data/influxdb/init-v1-compat.sh | 46 +++++++++++++++++++ 3 files changed, 55 insertions(+), 5 deletions(-) create mode 100644 data/influxdb/init-v1-compat.sh diff --git a/data/config/gsw_service.yaml.example b/data/config/gsw_service.yaml.example index 3da31f9..7d342b0 100644 --- a/data/config/gsw_service.yaml.example +++ b/data/config/gsw_service.yaml.example @@ -9,7 +9,7 @@ telemetry_config: data/config/backplane.yaml # If database_v2 is set, V2 will be used and V1 settings are ignored database_v2: url: http://localhost:8086 - token: your-token-here + token: gsw-local-dev-token org: gsw bucket: gsw batch_size: 100 # points buffered before auto-flush @@ -17,4 +17,4 @@ database_v2: precision: ns # ns | us | ms | s # Path to GSW service logging config -logging_config: data/config/logger.yaml \ No newline at end of file +logging_config: data/config/logger.yaml diff --git a/data/grafana/provisioning/datasources/gsw-influxdb-provider.yaml b/data/grafana/provisioning/datasources/gsw-influxdb-provider.yaml index 46767ec..48ec3ae 100644 --- a/data/grafana/provisioning/datasources/gsw-influxdb-provider.yaml +++ b/data/grafana/provisioning/datasources/gsw-influxdb-provider.yaml @@ -5,8 +5,12 @@ datasources: uid: gsw-influxdb type: influxdb access: proxy - - user: $GSW_GRAFANA_INFLUXDB_USER + url: $GSW_GRAFANA_INFLUXDB_URL + jsonData: + dbName: $GSW_GRAFANA_INFLUXDB_DB_NAME + httpHeaderName1: Authorization + httpMode: POST + version: InfluxQL secureJsonData: - password: $GSW_GRAFANA_INFLUXDB_PASSWORD + httpHeaderValue1: Token $GSW_GRAFANA_INFLUXDB_TOKEN diff --git a/data/influxdb/init-v1-compat.sh b/data/influxdb/init-v1-compat.sh new file mode 100644 index 0000000..56c3636 --- /dev/null +++ b/data/influxdb/init-v1-compat.sh @@ -0,0 +1,46 @@ +#!/bin/sh +set -eu + +host="${INFLUX_HOST:-http://influxdb:8086}" +org="${DOCKER_INFLUXDB_INIT_ORG:?DOCKER_INFLUXDB_INIT_ORG is required}" +bucket="${DOCKER_INFLUXDB_INIT_BUCKET:?DOCKER_INFLUXDB_INIT_BUCKET is required}" +token="${DOCKER_INFLUXDB_INIT_ADMIN_TOKEN:?DOCKER_INFLUXDB_INIT_ADMIN_TOKEN is required}" +database="${GSW_INFLUXDB_V1_DATABASE:-$bucket}" +retention_policy="${GSW_INFLUXDB_V1_RETENTION_POLICY:-autogen}" + +echo "Waiting for InfluxDB at ${host}" +until influx ping --host "${host}" >/dev/null 2>&1; do + sleep 1 +done + +bucket_id="$(influx bucket list \ + --host "${host}" \ + --org "${org}" \ + --token "${token}" \ + --name "${bucket}" \ + --hide-headers | awk 'NR == 1 { print $1 }')" + +if [ -z "${bucket_id}" ]; then + echo "Unable to resolve bucket ID for ${bucket}" >&2 + exit 1 +fi + +if influx v1 dbrp list \ + --host "${host}" \ + --org "${org}" \ + --token "${token}" \ + --db "${database}" \ + --hide-headers | grep -Eq "[[:space:]]${retention_policy}([[:space:]]|$)"; then + echo "DBRP mapping already exists for ${database}/${retention_policy}" + exit 0 +fi + +echo "Creating DBRP mapping for ${database}/${retention_policy}" +influx v1 dbrp create \ + --host "${host}" \ + --org "${org}" \ + --token "${token}" \ + --bucket-id "${bucket_id}" \ + --db "${database}" \ + --rp "${retention_policy}" \ + --default From ed827965f762262484e5f88cf566b6ced2485860 Mon Sep 17 00:00:00 2001 From: Aaron Chan Date: Fri, 3 Apr 2026 11:28:14 -0400 Subject: [PATCH 3/3] Add guide for querying influx over docker and improve config detection --- README.md | 15 ++++++--- cmd/gsw_service.go | 12 +++++-- cmd/gsw_service_test.go | 74 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+), 7 deletions(-) create mode 100644 cmd/gsw_service_test.go diff --git a/README.md b/README.md index 0f982db..9f5f840 100644 --- a/README.md +++ b/README.md @@ -57,12 +57,17 @@ If you need to change dashboards or datasources, set `GF_AUTH_ANONYMOUS_ORG_ROLE #### Accessing InfluxDB You can access the InfluxDB CLI using `docker compose exec -it influxdb influx`. -For example, to export all receiver telemetry as a CSV, run: +For example, to list buckets, run: ```shell -$ docker compose exec influxdb influx \ - -database "gsw" \ - -format csv \ - -execute "SELECT * FROM receiver" +docker compose exec influxdb influx bucket list +``` + +To query the `gsw` bucket directly, run: +```shell +docker compose exec influxdb influx query \ + --org gsw \ + --token gsw-local-dev-token \ + 'from(bucket: "gsw") |> range(start: -15m)' ``` ### Attaching to the container diff --git a/cmd/gsw_service.go b/cmd/gsw_service.go index 52c8eda..2924c70 100644 --- a/cmd/gsw_service.go +++ b/cmd/gsw_service.go @@ -140,8 +140,16 @@ func dbInitialize(ctx context.Context, channelMap map[int]chan []byte, cfg resol } func resolveDBConfig(config *viper.Viper) (resolvedDBConfig, error) { - v2Map := config.GetStringMap("database_v2") - if len(v2Map) > 0 { + v2Configured := config.IsSet("database_v2") || + config.IsSet("database_v2.url") || + config.IsSet("database_v2.token") || + config.IsSet("database_v2.org") || + config.IsSet("database_v2.bucket") || + config.IsSet("database_v2.batch_size") || + config.IsSet("database_v2.flush_interval_ms") || + config.IsSet("database_v2.precision") + + if v2Configured { precision, err := db.ParsePrecision(config.GetString("database_v2.precision")) if err != nil { return resolvedDBConfig{}, fmt.Errorf("invalid database_v2.precision: %w", err) diff --git a/cmd/gsw_service_test.go b/cmd/gsw_service_test.go new file mode 100644 index 0000000..1a33064 --- /dev/null +++ b/cmd/gsw_service_test.go @@ -0,0 +1,74 @@ +package main + +import ( + "strings" + "testing" + + "github.com/AarC10/GSW-V2/lib/db" + "github.com/spf13/viper" +) + +func newTestConfig() *viper.Viper { + cfg := viper.New() + cfg.SetEnvPrefix("GSW") + cfg.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + cfg.AutomaticEnv() + return cfg +} + +func TestResolveDBConfigFromEnvironmentUsesV2(t *testing.T) { + t.Setenv("GSW_DATABASE_V2_URL", "http://influxdb:8086") + t.Setenv("GSW_DATABASE_V2_TOKEN", "test-token") + t.Setenv("GSW_DATABASE_V2_ORG", "gsw") + t.Setenv("GSW_DATABASE_V2_BUCKET", "gsw") + t.Setenv("GSW_DATABASE_V2_BATCH_SIZE", "250") + t.Setenv("GSW_DATABASE_V2_FLUSH_INTERVAL_MS", "1500") + t.Setenv("GSW_DATABASE_V2_PRECISION", "ms") + + cfg := newTestConfig() + + got, err := resolveDBConfig(cfg) + if err != nil { + t.Fatalf("resolveDBConfig returned error: %v", err) + } + if got.v2 == nil { + t.Fatal("expected v2 config to be selected") + } + if got.v1 != nil { + t.Fatal("expected v1 config to remain unset") + } + + want := db.InfluxDBV2Config{ + URL: "http://influxdb:8086", + Token: "test-token", + Org: "gsw", + Bucket: "gsw", + BatchSize: 250, + FlushInterval: 1500, + Precision: db.PrecisionMS, + } + if *got.v2 != want { + t.Fatalf("got %+v, want %+v", *got.v2, want) + } +} + +func TestResolveDBConfigPrefersV2OverV1(t *testing.T) { + cfg := viper.New() + cfg.Set("database_host_name", "legacy") + cfg.Set("database_port_number", 8089) + cfg.Set("database_v2.url", "http://influxdb:8086") + cfg.Set("database_v2.token", "token") + cfg.Set("database_v2.org", "gsw") + cfg.Set("database_v2.bucket", "gsw") + + got, err := resolveDBConfig(cfg) + if err != nil { + t.Fatalf("resolveDBConfig returned error: %v", err) + } + if got.v2 == nil { + t.Fatal("expected v2 config to be selected") + } + if got.v1 != nil { + t.Fatal("expected v1 config to remain unset when v2 is configured") + } +}