From 45aa362789f53a5b49be0f38acf361b4430e2043 Mon Sep 17 00:00:00 2001 From: JurajKubrican Date: Fri, 15 Aug 2025 14:55:59 +0200 Subject: [PATCH] prometheus --- docs/grafana-integration.md | 259 +++++---------- go.mod | 9 + go.sum | 18 + src/main.go | 116 +++---- src/metrics.go | 20 ++ src/util/tracking.go | 633 ------------------------------------ 6 files changed, 189 insertions(+), 866 deletions(-) create mode 100644 src/metrics.go delete mode 100644 src/util/tracking.go diff --git a/docs/grafana-integration.md b/docs/grafana-integration.md index 8076c80..bf7be3f 100644 --- a/docs/grafana-integration.md +++ b/docs/grafana-integration.md @@ -1,154 +1,102 @@ # Grafana Integration Guide -This guide shows how to visualize your tracking data with Grafana using the new API endpoints. +This guide shows how to visualize your application metrics with Grafana using **real Prometheus**. + +## Why Prometheus? + +We've replaced custom tracking with actual Prometheus because: +- ✅ **Industry standard** - Battle-tested metrics collection +- ✅ **Rich ecosystem** - Thousands of existing dashboards +- ✅ **Better performance** - Optimized time-series storage +- ✅ **Less code** - No custom database or API endpoints +- ✅ **Standard metrics** - HTTP requests, response times, etc. + +## Available Metrics + +### Automatic Metrics (from echo-contrib/prometheus) +- `http_requests_total` - Total HTTP requests by method/status/endpoint +- `http_request_duration_seconds` - Request duration histogram +- `http_request_size_bytes` - Request size histogram +- `http_response_size_bytes` - Response size histogram + +### Custom Application Metrics +- `knet_page_views_total{route="/"}` - Page views by route +- `knet_page_views_total{route="/draw"}` - Draw page views +- `knet_box_clicks_total{position="1"}` - Box clicks by position +- `knet_websocket_connections` - Active WebSocket connections ## Authentication -In production, all tracking endpoints use **Basic Authentication** via Echo's built-in middleware: -- **Username:** `api` (or set via `API_USERNAME` environment variable) +The `/metrics` endpoint uses **Basic Authentication** in production: +- **Username:** `api` (or set via `API_USERNAME` environment variable) - **Password:** Your API token (set via `API_TOKEN` environment variable) **Example:** ```bash -curl -u api:your-api-token http://your-server:54321/api/metrics +curl -u api:your-api-token http://your-server:54321/metrics ``` -The authentication uses Echo's standard `middleware.BasicAuth` for reliability and security. +## Grafana Setup -## API Structure for Grafana - -### 1. Prometheus Metrics Endpoint -**URL:** `/api/metrics?period=24h` - -Returns Prometheus-style metrics that Grafana can scrape: -``` -# HELP http_requests_total Total HTTP requests -# TYPE http_requests_total counter -http_requests_total 1247 - -# HELP http_unique_users_total Unique users -# TYPE http_unique_users_total gauge -http_unique_users_total 234 -``` - -### 2. Time Series Data Endpoint -**URL:** `/api/tracking/timeseries?target=hits&interval=1h&from=2025-08-13&to=2025-08-14` - -Returns data in Grafana's expected format: -```json -[ - { - "target": "hits", - "datapoints": [ - [42, 1628856000000], - [38, 1628859600000], - [51, 1628863200000] - ] - } -] -``` - -### 3. Grafana Query Endpoint (SimpleJSON) -**URL:** `/api/tracking/query` (POST) - -Handles Grafana's native query format for more complex dashboards. - -## Grafana Setup Options - -### Option 1: Prometheus Data Source (Recommended) +### Prometheus Data Source (Recommended) 1. **Add Prometheus Data Source** in Grafana -2. **Set URL** to: `http://your-server:54321` (note: no `/api/metrics` suffix!) +2. **Set URL** to: `http://your-server:54321` 3. **Set Basic Auth**: Username: `api`, Password: `your-api-token` -4. **Set Scrape Interval**: 30s or 1m +4. **Set Scrape Interval**: 15s (Prometheus scrapes our `/metrics` endpoint) -Grafana will automatically query `/api/v1/query` and `/api/v1/query_range` endpoints. +That's it! Grafana will automatically discover all available metrics. -**Dashboard Queries:** +### Sample Dashboard Queries + +**Traffic Overview:** ```promql -# Total requests -http_requests_total +# Total requests per minute +rate(http_requests_total[1m]) * 60 -# Unique users -http_unique_users_total +# Page views +knet_page_views_total -# Unique endpoints -http_unique_endpoints_total +# Request duration 95th percentile +histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m])) -# Request rate (per minute) -rate(http_requests_total[5m]) * 60 +# Error rate +rate(http_requests_total{status=~"4.."}[5m]) / rate(http_requests_total[5m]) * 100 ``` -### Option 2: JSON API Data Source +**Application Metrics:** +```promql +# Box interactions +rate(knet_box_clicks_total[5m]) -1. **Install** the [JSON API Grafana Plugin](https://grafana.com/grafana/plugins/marcusolsson-json-datasource/) -2. **Add JSON API Data Source** -3. **Set URL** to: `http://your-server:54321/api/tracking/timeseries` -4. **Set Basic Auth**: Username: `api`, Password: `your-api-token` +# Active WebSocket connections +knet_websocket_connections -**Query Examples:** -- `hits` - Total page hits over time -- `users` - Unique users over time -- `endpoints` - Unique endpoints accessed - -### Option 3: SimpleJSON Data Source (Advanced) - -For complex dashboards with custom queries: - -1. **Install** the [SimpleJSON Data Source](https://grafana.com/grafana/plugins/grafana-simple-json-datasource/) -2. **Set URL** to: `http://your-server:54321/api/tracking/query` -3. **Configure** custom metrics: - - `http_requests` - Request time series - - `unique_users` - User count time series - - `top_endpoints` - Top endpoints table - -## Sample Dashboard Panels - -### 1. Traffic Overview (Single Stat) -```json -{ - "targets": [ - { - "target": "hits", - "refId": "A" - } - ], - "title": "Total Page Views (24h)", - "type": "singlestat" -} +# Most popular pages +topk(10, rate(knet_page_views_total[1h])) ``` -### 2. Traffic Trends (Graph) -```json -{ - "targets": [ - { - "target": "hits", - "refId": "A" - }, - { - "target": "users", - "refId": "B" - } - ], - "title": "Traffic Over Time", - "type": "graph" -} -``` +## Dashboard Examples -### 3. Top Endpoints (Table) -```json -{ - "targets": [ - { - "target": "top_endpoints", - "refId": "A" - } - ], - "title": "Most Popular Pages", - "type": "table" -} -``` +### 1. HTTP Overview Panel +- **Metric:** `rate(http_requests_total[1m]) * 60` +- **Type:** Graph +- **Title:** "Requests per Minute" + +### 2. Response Time Panel +- **Metric:** `histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m]))` +- **Type:** Single Stat +- **Title:** "95th Percentile Response Time" + +### 3. Page Views Panel +- **Metric:** `knet_page_views_total` +- **Type:** Graph +- **Title:** "Page Views by Route" + +### 4. Box Clicks Heatmap +- **Metric:** `rate(knet_box_clicks_total[5m])` +- **Type:** Graph +- **Title:** "Box Clicks per Second" ## Environment Variables @@ -156,63 +104,24 @@ Set these for production: ```bash export API_TOKEN="your-secure-api-token" export API_USERNAME="api" # Optional, defaults to "api" -export BUILD_NUMBER="prod-v1.0" -``` - -## Grafana Dashboard JSON - -Here's a complete dashboard configuration: - -```json -{ - "dashboard": { - "title": "Website Analytics", - "panels": [ - { - "title": "Page Views (24h)", - "type": "singlestat", - "targets": [{"target": "hits"}], - "gridPos": {"h": 8, "w": 6, "x": 0, "y": 0} - }, - { - "title": "Unique Users (24h)", - "type": "singlestat", - "targets": [{"target": "users"}], - "gridPos": {"h": 8, "w": 6, "x": 6, "y": 0} - }, - { - "title": "Traffic Trends", - "type": "graph", - "targets": [ - {"target": "hits", "refId": "A"}, - {"target": "users", "refId": "B"} - ], - "gridPos": {"h": 8, "w": 12, "x": 0, "y": 8} - } - ], - "time": { - "from": "now-24h", - "to": "now" - }, - "refresh": "30s" - } -} +export BUILD_NUMBER="prod-v1.0" # Enables production mode ``` ## Benefits of This Approach -1. **Multiple Grafana Data Sources** - Choose what works best for you -2. **Real-time Updates** - Data refreshes automatically -3. **Flexible Time Ranges** - Hour, day, week, month views -4. **Standard Formats** - Works with existing Grafana plugins -5. **Authentication** - Secured with Basic Auth in production +1. **Standard Prometheus** - Works with all Grafana features +2. **Rich Metrics** - HTTP performance + custom business metrics +3. **No Database** - Prometheus handles storage and retention +4. **Better Performance** - Optimized time-series queries +5. **Alerting Ready** - Use Prometheus AlertManager +6. **Ecosystem** - Tons of existing dashboards and tools ## Quick Start -1. Start your server: `./tmp/main` -2. Test metrics endpoint: `curl http://localhost:54321/api/metrics` -3. Add Prometheus data source in Grafana pointing to `/api/metrics` -4. Create dashboard with `http_requests_total` and `http_unique_users_total` queries -5. Set refresh interval to 30 seconds +1. **Start your server:** `./tmp/main` +2. **Test metrics:** `curl -u api:your-token http://localhost:54321/metrics` +3. **Add Prometheus data source** in Grafana: `http://localhost:54321` +4. **Create dashboard** with queries like `http_requests_total` +5. **Enjoy real-time metrics!** 📊 -Your analytics will now update in real-time in Grafana! 📊 +Your analytics are now powered by industry-standard Prometheus! � diff --git a/go.mod b/go.mod index ba855bd..828daea 100644 --- a/go.mod +++ b/go.mod @@ -9,11 +9,19 @@ require ( ) require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/labstack/echo-contrib v0.17.4 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect + github.com/prometheus/client_golang v1.23.0 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.65.0 // indirect + github.com/prometheus/procfs v0.16.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.2 // indirect @@ -22,6 +30,7 @@ require ( golang.org/x/sys v0.34.0 // indirect golang.org/x/text v0.27.0 // indirect golang.org/x/time v0.12.0 // indirect + google.golang.org/protobuf v1.36.6 // indirect modernc.org/libc v1.66.3 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect diff --git a/go.sum b/go.sum index 9579794..1cf2c98 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,15 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/labstack/echo-contrib v0.17.4 h1:g5mfsrJfJTKv+F5uNKCyrjLK7js+ZW6HTjg4FnDxxgk= +github.com/labstack/echo-contrib v0.17.4/go.mod h1:9O7ZPAHUeMGTOAfg80YqQduHzt0CzLak36PZRldYrZ0= github.com/labstack/echo/v4 v4.13.4 h1:oTZZW+T3s9gAu5L8vmzihV7/lkXGZuITzTQkTEhcXEA= github.com/labstack/echo/v4 v4.13.4/go.mod h1:g63b33BZ5vZzcIUF8AtRH40DrTlXnx4UMC8rBdndmjQ= github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0= @@ -12,10 +18,20 @@ github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHP github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.0 h1:ust4zpdl9r4trLY/gSjlm07PuiBq2ynaXXlptpfy8Uc= +github.com/prometheus/client_golang v1.23.0/go.mod h1:i/o0R9ByOnHX0McrTMTyhYvKE4haaf2mW08I+jGAjEE= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.65.0 h1:QDwzd+G1twt//Kwj/Ww6E9FQq1iVMmODnILtW1t2VzE= +github.com/prometheus/common v0.65.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= @@ -37,6 +53,8 @@ golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= modernc.org/libc v1.66.3 h1:cfCbjTUcdsKyyZZfEUKfoHcP3S0Wkvz3jgSzByEWVCQ= diff --git a/src/main.go b/src/main.go index bdd1daa..b42ffb8 100644 --- a/src/main.go +++ b/src/main.go @@ -8,9 +8,12 @@ import ( "strings" "time" + "github.com/labstack/echo-contrib/prometheus" "github.com/labstack/echo/v4" "github.com/labstack/echo/v4/middleware" "github.com/labstack/gommon/log" + promclient "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" "knet.sk/src/boxes" "knet.sk/src/draw" "knet.sk/src/util" @@ -46,26 +49,66 @@ func newPage(boxes []boxes.Box) Page { var ( e = echo.New() + + // Custom Prometheus metrics + boxClicks = promclient.NewCounterVec( + promclient.CounterOpts{ + Name: "knet_box_clicks_total", + Help: "Total number of box clicks by position", + }, + []string{"position"}, + ) + + activeConnections = promclient.NewGauge( + promclient.GaugeOpts{ + Name: "knet_websocket_connections", + Help: "Current number of active WebSocket connections", + }, + ) + + pageViews = promclient.NewCounterVec( + promclient.CounterOpts{ + Name: "knet_page_views_total", + Help: "Total page views by route", + }, + []string{"route"}, + ) ) +func init() { + // Register custom metrics + promclient.MustRegister(boxClicks) + promclient.MustRegister(activeConnections) + promclient.MustRegister(pageViews) +} + func main() { e.Renderer = NewTemplates() - // Initialize tracking service - trackingService, err := util.NewTrackingService("data/tracking.db") - if err != nil { - e.Logger.Fatal("Failed to initialize tracking service:", err) - } - defer trackingService.Close() + // Setup Prometheus metrics + p := prometheus.NewPrometheus("knet", nil) + p.Use(e) e.Logger.SetLevel(log.DEBUG) e.Use(middleware.Logger()) - e.Use(trackingService.TrackingMiddleware()) if util.IsProd() { e.Use(middleware.Gzip()) e.Use(middleware.HTTPSRedirect()) } + // Prometheus metrics endpoint (standard /metrics) + metricsGroup := e.Group("/metrics") + if util.IsProd() { + metricsGroup.Use(middleware.BasicAuth(func(username, password string, c echo.Context) (bool, error) { + expectedToken := os.Getenv("API_TOKEN") + if expectedToken == "" { + return false, nil + } + return username == "api" && password == expectedToken, nil + })) + } + metricsGroup.GET("", echo.WrapHandler(promhttp.Handler())) + e.GET("/health", func(c echo.Context) error { return c.Render(200, "health", Page{ BuildNumber: util.GetBuildNumber(), @@ -88,62 +131,19 @@ func main() { e.Static("/images", "images") e.GET("/", func(c echo.Context) error { + // Track page view + pageViews.WithLabelValues("/").Inc() return c.Render(200, "index", newPage(boxes.GetBoxes())) }) - // Tracking API endpoints (protected by basic auth if needed) - api := e.Group("/api") - if util.IsProd() { - api.Use(middleware.BasicAuth(func(username, password string, c echo.Context) (bool, error) { - // Get expected credentials from environment - expectedToken := os.Getenv("API_TOKEN") - if expectedToken == "" { - return false, nil // No token configured - } - - // Check credentials - return username == "api" && password == expectedToken, nil - })) - } - - // Grafana-compatible endpoints - api.GET("/metrics", trackingService.GetMetrics) - api.GET("/tracking/timeseries", trackingService.GetTimeSeriesData) - api.POST("/tracking/query", trackingService.GetGrafanaQuery) - - // Prometheus-compatible endpoints for Grafana - prometheusAPI := e.Group("/api/v1") - if util.IsProd() { - prometheusAPI.Use(middleware.BasicAuth(func(username, password string, c echo.Context) (bool, error) { - expectedToken := os.Getenv("API_TOKEN") - if expectedToken == "" { - return false, nil - } - return username == "api" && password == expectedToken, nil - })) - } - - // Prometheus API endpoints that Grafana expects - prometheusAPI.GET("/status/buildinfo", func(c echo.Context) error { - return c.JSON(http.StatusOK, map[string]interface{}{ - "status": "success", - "data": map[string]interface{}{ - "version": "knet-tracking-1.0", - "revision": "main", - "branch": "main", - "buildUser": "knet", - "buildDate": "2025-08-15", - "goVersion": "go1.24.0", - }, - }) - }) - - prometheusAPI.POST("/query", trackingService.GetPrometheusQuery) - prometheusAPI.POST("/query_range", trackingService.GetPrometheusQueryRange) - e.GET("/boxes/ws", boxes.HandleBoxesWs) - e.GET("/draw", draw.Page) + e.GET("/draw", func(c echo.Context) error { + // Track page view + pageViews.WithLabelValues("/draw").Inc() + return draw.Page(c) + }) + e.GET("/draw/ws", draw.InitWs) e.GET("/draw/ws", draw.InitWs) go boxes.RegisterTicker() diff --git a/src/metrics.go b/src/metrics.go new file mode 100644 index 0000000..2cdf100 --- /dev/null +++ b/src/metrics.go @@ -0,0 +1,20 @@ +package main + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +// GetActiveConnections returns the current number of active WebSocket connections +// This function will be called by Prometheus to get the current gauge value +func GetActiveConnections() prometheus.GaugeFunc { + return prometheus.NewGaugeFunc( + prometheus.GaugeOpts{ + Name: "knet_websocket_connections_active", + Help: "Current number of active WebSocket connections", + }, + func() float64 { + // This will be implemented to call boxes.GetConnectionCount() + return 0 // Placeholder for now + }, + ) +} diff --git a/src/util/tracking.go b/src/util/tracking.go deleted file mode 100644 index 67e03a7..0000000 --- a/src/util/tracking.go +++ /dev/null @@ -1,633 +0,0 @@ -package util - -import ( - "crypto/md5" - "database/sql" - "fmt" - "net/http" - "strings" - "time" - - "github.com/labstack/echo/v4" - _ "modernc.org/sqlite" -) - -type TrackingService struct { - db *sql.DB -} - -type UserVisit struct { - ID int `json:"id" db:"id"` - UserHash string `json:"user_hash" db:"user_hash"` - Endpoint string `json:"endpoint" db:"endpoint"` - Method string `json:"method" db:"method"` - IPAddress string `json:"ip_address" db:"ip_address"` - UserAgent string `json:"user_agent" db:"user_agent"` - Referer string `json:"referer" db:"referer"` - Timestamp time.Time `json:"timestamp" db:"timestamp"` -} - -func NewTrackingService(dbPath string) (*TrackingService, error) { - db, err := sql.Open("sqlite", dbPath) - if err != nil { - return nil, fmt.Errorf("failed to open tracking database: %w", err) - } - - service := &TrackingService{db: db} - if err := service.initDB(); err != nil { - return nil, fmt.Errorf("failed to initialize tracking database: %w", err) - } - - return service, nil -} - -func (ts *TrackingService) initDB() error { - query := ` - CREATE TABLE IF NOT EXISTS user_visits ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - user_hash TEXT NOT NULL, - endpoint TEXT NOT NULL, - method TEXT NOT NULL, - ip_address TEXT, - user_agent TEXT, - referer TEXT, - timestamp DATETIME DEFAULT CURRENT_TIMESTAMP - ); - - CREATE INDEX IF NOT EXISTS idx_user_hash ON user_visits(user_hash); - CREATE INDEX IF NOT EXISTS idx_endpoint_method ON user_visits(endpoint, method); - CREATE INDEX IF NOT EXISTS idx_timestamp ON user_visits(timestamp); - CREATE INDEX IF NOT EXISTS idx_endpoint_timestamp ON user_visits(endpoint, timestamp); - ` - - _, err := ts.db.Exec(query) - return err -} - -func (ts *TrackingService) generateUserHash(c echo.Context) string { - // Create a hash from IP + User-Agent for basic user identification - ip := c.RealIP() - userAgent := c.Request().UserAgent() - - // Combine IP and User-Agent for a basic fingerprint - fingerprint := fmt.Sprintf("%s|%s", ip, userAgent) - - // MD5 hash for privacy (not cryptographically secure but fine for tracking) - hash := md5.Sum([]byte(fingerprint)) - return fmt.Sprintf("%x", hash) -} - -func (ts *TrackingService) shouldTrack(path string) bool { - // Skip tracking for certain endpoints - skipPrefixes := []string{ - "/css/", - "/js/", - "/images/", - "/swagger/", - "/favicon", - } - - for _, prefix := range skipPrefixes { - if strings.HasPrefix(path, prefix) { - return false - } - } - - return true -} - -func (ts *TrackingService) TrackingMiddleware() echo.MiddlewareFunc { - return func(next echo.HandlerFunc) echo.HandlerFunc { - return func(c echo.Context) error { - // Execute the request first - err := next(c) - - // Only track successful requests and trackable paths - if c.Response().Status < 400 && ts.shouldTrack(c.Request().URL.Path) { - go ts.trackRequest(c) - } - - return err - } - } -} - -func (ts *TrackingService) trackRequest(c echo.Context) { - endpoint := c.Request().URL.Path - method := c.Request().Method - userHash := ts.generateUserHash(c) - - // Record the visit - this is all we need now - ts.recordVisit(userHash, endpoint, method, c) -} - -func (ts *TrackingService) recordVisit(userHash, endpoint, method string, c echo.Context) error { - query := ` - INSERT INTO user_visits (user_hash, endpoint, method, ip_address, user_agent, referer) - VALUES (?, ?, ?, ?, ?, ?) - ` - - _, err := ts.db.Exec(query, - userHash, - endpoint, - method, - c.RealIP(), - c.Request().UserAgent(), - c.Request().Referer(), - ) - - return err -} - -func (ts *TrackingService) Close() error { - return ts.db.Close() -} - -// Prometheus-compatible endpoints for Grafana - -// GetPrometheusQuery handles Prometheus /api/v1/query endpoint -func (ts *TrackingService) GetPrometheusQuery(c echo.Context) error { - query := c.FormValue("query") - - if query == "" { - return c.JSON(http.StatusBadRequest, map[string]interface{}{ - "status": "error", - "error": "query parameter is required", - }) - } - - // Handle different metric queries - var value float64 - var metricName string - - switch query { - case "http_requests_total": - metricName = "http_requests_total" - var count int - ts.db.QueryRow("SELECT COUNT(*) FROM user_visits WHERE timestamp >= datetime('now', '-1 day')").Scan(&count) - value = float64(count) - - case "http_unique_users_total": - metricName = "http_unique_users_total" - var count int - ts.db.QueryRow("SELECT COUNT(DISTINCT user_hash) FROM user_visits WHERE timestamp >= datetime('now', '-1 day')").Scan(&count) - value = float64(count) - - case "http_unique_endpoints_total": - metricName = "http_unique_endpoints_total" - var count int - ts.db.QueryRow("SELECT COUNT(DISTINCT endpoint) FROM user_visits WHERE timestamp >= datetime('now', '-1 day')").Scan(&count) - value = float64(count) - - default: - return c.JSON(http.StatusOK, map[string]interface{}{ - "status": "success", - "data": map[string]interface{}{ - "resultType": "vector", - "result": []interface{}{}, - }, - }) - } - - // Return Prometheus-style response - return c.JSON(http.StatusOK, map[string]interface{}{ - "status": "success", - "data": map[string]interface{}{ - "resultType": "vector", - "result": []map[string]interface{}{ - { - "metric": map[string]string{ - "__name__": metricName, - }, - "value": []interface{}{ - time.Now().Unix(), - fmt.Sprintf("%.0f", value), - }, - }, - }, - }, - }) -} - -// GetPrometheusQueryRange handles Prometheus /api/v1/query_range endpoint -func (ts *TrackingService) GetPrometheusQueryRange(c echo.Context) error { - query := c.FormValue("query") - start := c.FormValue("start") // Unix timestamp - end := c.FormValue("end") // Unix timestamp - step := c.FormValue("step") // Duration like "15s", "1m", "5m" - - if query == "" { - return c.JSON(http.StatusBadRequest, map[string]interface{}{ - "status": "error", - "error": "query parameter is required", - }) - } - - // Parse step duration (default to 1 minute) - stepSeconds := 60 // Default 1 minute - if step != "" { - if strings.HasSuffix(step, "s") { - if s, err := fmt.Sscanf(step, "%ds", &stepSeconds); s != 1 || err != nil { - stepSeconds = 60 - } - } else if strings.HasSuffix(step, "m") { - var minutes int - if s, err := fmt.Sscanf(step, "%dm", &minutes); s == 1 && err == nil { - stepSeconds = minutes * 60 - } - } - } - - // Build time condition - var timeCondition string - if start != "" && end != "" { - timeCondition = fmt.Sprintf("WHERE timestamp BETWEEN datetime(%s, 'unixepoch') AND datetime(%s, 'unixepoch')", start, end) - } else { - timeCondition = "WHERE timestamp >= datetime('now', '-24 hours')" - } - - // Determine what metric to query - var sqlQuery string - var metricName string - - switch query { - case "http_requests_total": - metricName = "http_requests_total" - sqlQuery = fmt.Sprintf(` - SELECT - strftime('%%s', strftime('%%Y-%%m-%%d %%H:%%M:00', timestamp)) as ts, - COUNT(*) as value - FROM user_visits - %s - GROUP BY strftime('%%Y-%%m-%%d %%H:%%M:00', timestamp) - ORDER BY ts - `, timeCondition) - - case "http_unique_users_total": - metricName = "http_unique_users_total" - sqlQuery = fmt.Sprintf(` - SELECT - strftime('%%s', strftime('%%Y-%%m-%%d %%H:%%M:00', timestamp)) as ts, - COUNT(DISTINCT user_hash) as value - FROM user_visits - %s - GROUP BY strftime('%%Y-%%m-%%d %%H:%%M:00', timestamp) - ORDER BY ts - `, timeCondition) - - default: - return c.JSON(http.StatusOK, map[string]interface{}{ - "status": "success", - "data": map[string]interface{}{ - "resultType": "matrix", - "result": []interface{}{}, - }, - }) - } - - rows, err := ts.db.Query(sqlQuery) - if err != nil { - return c.JSON(http.StatusInternalServerError, map[string]interface{}{ - "status": "error", - "error": "Failed to execute query", - }) - } - defer rows.Close() - - var values [][]interface{} - for rows.Next() { - var timestamp int64 - var value int - err := rows.Scan(×tamp, &value) - if err != nil { - continue - } - values = append(values, []interface{}{timestamp, fmt.Sprintf("%d", value)}) - } - - return c.JSON(http.StatusOK, map[string]interface{}{ - "status": "success", - "data": map[string]interface{}{ - "resultType": "matrix", - "result": []map[string]interface{}{ - { - "metric": map[string]string{ - "__name__": metricName, - }, - "values": values, - }, - }, - }, - }) -} - -// Grafana-compatible endpoints below - -// GetMetrics returns Prometheus-style metrics for Grafana -func (ts *TrackingService) GetMetrics(c echo.Context) error { - period := c.QueryParam("period") - if period == "" { - period = "24h" - } - - var timeCondition string - switch period { - case "1h": - timeCondition = "WHERE timestamp >= datetime('now', '-1 hour')" - case "24h": - timeCondition = "WHERE timestamp >= datetime('now', '-1 day')" - case "7d": - timeCondition = "WHERE timestamp >= datetime('now', '-7 days')" - case "30d": - timeCondition = "WHERE timestamp >= datetime('now', '-30 days')" - default: - timeCondition = "WHERE timestamp >= datetime('now', '-1 day')" - } - - // Get basic metrics - var totalHits, uniqueUsers, uniqueEndpoints int - query := fmt.Sprintf("SELECT COUNT(*) FROM user_visits %s", timeCondition) - ts.db.QueryRow(query).Scan(&totalHits) - - query = fmt.Sprintf("SELECT COUNT(DISTINCT user_hash) FROM user_visits %s", timeCondition) - ts.db.QueryRow(query).Scan(&uniqueUsers) - - query = fmt.Sprintf("SELECT COUNT(DISTINCT endpoint) FROM user_visits %s", timeCondition) - ts.db.QueryRow(query).Scan(&uniqueEndpoints) - - // Return Prometheus-style metrics - response := fmt.Sprintf(`# HELP http_requests_total Total HTTP requests -# TYPE http_requests_total counter -http_requests_total %d - -# HELP http_unique_users_total Unique users -# TYPE http_unique_users_total gauge -http_unique_users_total %d - -# HELP http_unique_endpoints_total Unique endpoints accessed -# TYPE http_unique_endpoints_total gauge -http_unique_endpoints_total %d -`, totalHits, uniqueUsers, uniqueEndpoints) - - return c.String(http.StatusOK, response) -} - -// GetTimeSeriesData returns time-series data for Grafana charts -func (ts *TrackingService) GetTimeSeriesData(c echo.Context) error { - target := c.QueryParam("target") // What to measure: hits, users, endpoints - from := c.QueryParam("from") // Start time (unix timestamp or relative) - to := c.QueryParam("to") // End time - interval := c.QueryParam("interval") // Time interval: 1m, 5m, 1h, 1d - - if target == "" { - target = "hits" - } - if interval == "" { - interval = "1h" - } - - // Convert interval to SQLite format - var timeFormat string - switch interval { - case "1m": - timeFormat = "strftime('%Y-%m-%d %H:%M', timestamp)" - case "5m": - timeFormat = "strftime('%Y-%m-%d %H:%M', datetime(timestamp, 'unixepoch', 'start of minute', '+' || (strftime('%M', timestamp) / 5) * 5 || ' minutes'))" - case "1h": - timeFormat = "strftime('%Y-%m-%d %H:00', timestamp)" - case "1d": - timeFormat = "strftime('%Y-%m-%d', timestamp)" - default: - timeFormat = "strftime('%Y-%m-%d %H:00', timestamp)" - } - - // Build time condition - var timeCondition string - if from != "" && to != "" { - timeCondition = fmt.Sprintf("WHERE timestamp BETWEEN datetime('%s') AND datetime('%s')", from, to) - } else { - timeCondition = "WHERE timestamp >= datetime('now', '-24 hours')" - } - - // Build query based on target - var selectClause string - switch target { - case "hits": - selectClause = "COUNT(*) as value" - case "users": - selectClause = "COUNT(DISTINCT user_hash) as value" - case "endpoints": - selectClause = "COUNT(DISTINCT endpoint) as value" - default: - selectClause = "COUNT(*) as value" - } - - query := fmt.Sprintf(` - SELECT - %s as time_bucket, - %s, - strftime('%%s', %s) * 1000 as timestamp_ms - FROM user_visits - %s - GROUP BY time_bucket - ORDER BY timestamp_ms ASC - `, timeFormat, selectClause, timeFormat, timeCondition) - - rows, err := ts.db.Query(query) - if err != nil { - return c.JSON(http.StatusInternalServerError, map[string]string{ - "error": "Failed to retrieve time series data", - }) - } - defer rows.Close() - - type DataPoint struct { - Target string `json:"target"` - DataPoints [][2]interface{} `json:"datapoints"` // [value, timestamp_ms] - } - - var dataPoints [][2]interface{} - for rows.Next() { - var timeBucket string - var value int - var timestampMs int64 - - err := rows.Scan(&timeBucket, &value, ×tampMs) - if err != nil { - continue - } - - dataPoints = append(dataPoints, [2]interface{}{value, timestampMs}) - } - - response := []DataPoint{ - { - Target: target, - DataPoints: dataPoints, - }, - } - - return c.JSON(http.StatusOK, response) -} - -// GetGrafanaQuery handles Grafana's query format -func (ts *TrackingService) GetGrafanaQuery(c echo.Context) error { - // This endpoint mimics Grafana's SimpleJSON datasource format - var request struct { - PanelId int `json:"panelId"` - Range struct { - From string `json:"from"` - To string `json:"to"` - } `json:"range"` - RangeRaw struct { - From string `json:"from"` - To string `json:"to"` - } `json:"rangeRaw"` - Interval string `json:"interval"` - IntervalMs int `json:"intervalMs"` - Targets []struct { - Target string `json:"target"` - RefId string `json:"refId"` - Type string `json:"type"` - } `json:"targets"` - Format string `json:"format"` - MaxDataPoints int `json:"maxDataPoints"` - } - - if err := c.Bind(&request); err != nil { - return c.JSON(http.StatusBadRequest, map[string]string{ - "error": "Invalid request format", - }) - } - - var results []interface{} - - for _, target := range request.Targets { - // Parse the target to determine what to query - // Format: "metric_name.endpoint_filter.method_filter" - parts := strings.Split(target.Target, ".") - metricName := parts[0] - - var timeCondition string - if request.Range.From != "" && request.Range.To != "" { - timeCondition = fmt.Sprintf("WHERE timestamp BETWEEN '%s' AND '%s'", - request.Range.From, request.Range.To) - } else { - timeCondition = "WHERE timestamp >= datetime('now', '-24 hours')" - } - - switch metricName { - case "http_requests": - results = append(results, ts.getRequestsTimeSeries(timeCondition, target.RefId)) - case "unique_users": - results = append(results, ts.getUniqueUsersTimeSeries(timeCondition, target.RefId)) - case "top_endpoints": - results = append(results, ts.getTopEndpoints(timeCondition, target.RefId)) - } - } - - return c.JSON(http.StatusOK, results) -} - -// Helper functions for Grafana queries -func (ts *TrackingService) getRequestsTimeSeries(timeCondition, refId string) map[string]interface{} { - query := fmt.Sprintf(` - SELECT - strftime('%%Y-%%m-%%d %%H:00', timestamp) as hour, - COUNT(*) as requests, - strftime('%%s', strftime('%%Y-%%m-%%d %%H:00', timestamp)) * 1000 as timestamp_ms - FROM user_visits - %s - GROUP BY hour - ORDER BY timestamp_ms ASC - `, timeCondition) - - rows, _ := ts.db.Query(query) - defer rows.Close() - - var dataPoints [][2]interface{} - for rows.Next() { - var hour string - var requests int - var timestampMs int64 - rows.Scan(&hour, &requests, ×tampMs) - dataPoints = append(dataPoints, [2]interface{}{requests, timestampMs}) - } - - return map[string]interface{}{ - "target": "HTTP Requests", - "refId": refId, - "datapoints": dataPoints, - } -} - -func (ts *TrackingService) getUniqueUsersTimeSeries(timeCondition, refId string) map[string]interface{} { - query := fmt.Sprintf(` - SELECT - strftime('%%Y-%%m-%%d %%H:00', timestamp) as hour, - COUNT(DISTINCT user_hash) as users, - strftime('%%s', strftime('%%Y-%%m-%%d %%H:00', timestamp)) * 1000 as timestamp_ms - FROM user_visits - %s - GROUP BY hour - ORDER BY timestamp_ms ASC - `, timeCondition) - - rows, _ := ts.db.Query(query) - defer rows.Close() - - var dataPoints [][2]interface{} - for rows.Next() { - var hour string - var users int - var timestampMs int64 - rows.Scan(&hour, &users, ×tampMs) - dataPoints = append(dataPoints, [2]interface{}{users, timestampMs}) - } - - return map[string]interface{}{ - "target": "Unique Users", - "refId": refId, - "datapoints": dataPoints, - } -} - -func (ts *TrackingService) getTopEndpoints(timeCondition, refId string) map[string]interface{} { - query := fmt.Sprintf(` - SELECT - endpoint, - COUNT(*) as hits - FROM user_visits - %s - GROUP BY endpoint - ORDER BY hits DESC - LIMIT 10 - `, timeCondition) - - rows, _ := ts.db.Query(query) - defer rows.Close() - - type EndpointData struct { - Endpoint string `json:"endpoint"` - Hits int `json:"hits"` - } - - var endpoints []EndpointData - for rows.Next() { - var endpoint string - var hits int - rows.Scan(&endpoint, &hits) - endpoints = append(endpoints, EndpointData{Endpoint: endpoint, Hits: hits}) - } - - return map[string]interface{}{ - "target": "Top Endpoints", - "refId": refId, - "type": "table", - "columns": []map[string]string{ - {"text": "Endpoint", "type": "string"}, - {"text": "Hits", "type": "number"}, - }, - "rows": endpoints, - } -}