This commit is contained in:
JurajKubrican
2025-08-15 14:12:15 +02:00
parent e3c6500b06
commit fecbe91f5d
6 changed files with 680 additions and 227 deletions

View File

@@ -4,6 +4,7 @@ import (
"html/template"
"io"
"net/http"
"os"
"strings"
"time"
@@ -90,15 +91,55 @@ func main() {
return c.Render(200, "index", newPage(boxes.GetBoxes()))
})
// Tracking API endpoints (protected by bearer auth if needed)
// Tracking API endpoints (protected by basic auth if needed)
api := e.Group("/api")
if util.IsProd() {
api.Use(util.BearerAuthMiddleware())
api.Use(middleware.BasicAuth(func(username, password string, c echo.Context) (bool, error) {
// Get expected credentials from environment
expectedToken := os.Getenv("API_TOKEN")
if expectedToken == "" {
return false, nil // No token configured
}
// Check credentials
return username == "api" && password == expectedToken, nil
}))
}
api.GET("/tracking/stats", trackingService.GetEndpointStats)
api.GET("/tracking/visits", trackingService.GetRecentVisits)
api.GET("/tracking/summary", trackingService.GetTrackingSummary)
api.GET("/tracking/trends", trackingService.GetTimeTrends)
// Grafana-compatible endpoints
api.GET("/metrics", trackingService.GetMetrics)
api.GET("/tracking/timeseries", trackingService.GetTimeSeriesData)
api.POST("/tracking/query", trackingService.GetGrafanaQuery)
// Prometheus-compatible endpoints for Grafana
prometheusAPI := e.Group("/api/v1")
if util.IsProd() {
prometheusAPI.Use(middleware.BasicAuth(func(username, password string, c echo.Context) (bool, error) {
expectedToken := os.Getenv("API_TOKEN")
if expectedToken == "" {
return false, nil
}
return username == "api" && password == expectedToken, nil
}))
}
// Prometheus API endpoints that Grafana expects
prometheusAPI.GET("/status/buildinfo", func(c echo.Context) error {
return c.JSON(http.StatusOK, map[string]interface{}{
"status": "success",
"data": map[string]interface{}{
"version": "knet-tracking-1.0",
"revision": "main",
"branch": "main",
"buildUser": "knet",
"buildDate": "2025-08-15",
"goVersion": "go1.24.0",
},
})
})
prometheusAPI.POST("/query", trackingService.GetPrometheusQuery)
prometheusAPI.POST("/query_range", trackingService.GetPrometheusQueryRange)
e.GET("/boxes/ws", boxes.HandleBoxesWs)

View File

@@ -139,263 +139,495 @@ func (ts *TrackingService) recordVisit(userHash, endpoint, method string, c echo
return err
}
// GetEndpointStats returns statistics for all endpoints with time period support
func (ts *TrackingService) GetEndpointStats(c echo.Context) error {
period := c.QueryParam("period") // day, week, month, year, all
if period == "" {
period = "all"
func (ts *TrackingService) Close() error {
return ts.db.Close()
}
// Prometheus-compatible endpoints for Grafana
// GetPrometheusQuery handles Prometheus /api/v1/query endpoint
func (ts *TrackingService) GetPrometheusQuery(c echo.Context) error {
query := c.FormValue("query")
if query == "" {
return c.JSON(http.StatusBadRequest, map[string]interface{}{
"status": "error",
"error": "query parameter is required",
})
}
var timeCondition string
switch period {
case "day":
timeCondition = "AND timestamp >= datetime('now', '-1 day')"
case "week":
timeCondition = "AND timestamp >= datetime('now', '-7 days')"
case "month":
timeCondition = "AND timestamp >= datetime('now', '-1 month')"
case "year":
timeCondition = "AND timestamp >= datetime('now', '-1 year')"
// Handle different metric queries
var value float64
var metricName string
switch query {
case "http_requests_total":
metricName = "http_requests_total"
var count int
ts.db.QueryRow("SELECT COUNT(*) FROM user_visits WHERE timestamp >= datetime('now', '-1 day')").Scan(&count)
value = float64(count)
case "http_unique_users_total":
metricName = "http_unique_users_total"
var count int
ts.db.QueryRow("SELECT COUNT(DISTINCT user_hash) FROM user_visits WHERE timestamp >= datetime('now', '-1 day')").Scan(&count)
value = float64(count)
case "http_unique_endpoints_total":
metricName = "http_unique_endpoints_total"
var count int
ts.db.QueryRow("SELECT COUNT(DISTINCT endpoint) FROM user_visits WHERE timestamp >= datetime('now', '-1 day')").Scan(&count)
value = float64(count)
default:
timeCondition = ""
return c.JSON(http.StatusOK, map[string]interface{}{
"status": "success",
"data": map[string]interface{}{
"resultType": "vector",
"result": []interface{}{},
},
})
}
query := fmt.Sprintf(`
SELECT
endpoint,
method,
COUNT(*) as hit_count,
COUNT(DISTINCT user_hash) as unique_users,
MAX(timestamp) as last_access,
MIN(timestamp) as first_access
FROM user_visits
WHERE 1=1 %s
GROUP BY endpoint, method
ORDER BY hit_count DESC
`, timeCondition)
// Return Prometheus-style response
return c.JSON(http.StatusOK, map[string]interface{}{
"status": "success",
"data": map[string]interface{}{
"resultType": "vector",
"result": []map[string]interface{}{
{
"metric": map[string]string{
"__name__": metricName,
},
"value": []interface{}{
time.Now().Unix(),
fmt.Sprintf("%.0f", value),
},
},
},
},
})
}
rows, err := ts.db.Query(query)
// GetPrometheusQueryRange handles Prometheus /api/v1/query_range endpoint
func (ts *TrackingService) GetPrometheusQueryRange(c echo.Context) error {
query := c.FormValue("query")
start := c.FormValue("start") // Unix timestamp
end := c.FormValue("end") // Unix timestamp
step := c.FormValue("step") // Duration like "15s", "1m", "5m"
if query == "" {
return c.JSON(http.StatusBadRequest, map[string]interface{}{
"status": "error",
"error": "query parameter is required",
})
}
// Parse step duration (default to 1 minute)
stepSeconds := 60 // Default 1 minute
if step != "" {
if strings.HasSuffix(step, "s") {
if s, err := fmt.Sscanf(step, "%ds", &stepSeconds); s != 1 || err != nil {
stepSeconds = 60
}
} else if strings.HasSuffix(step, "m") {
var minutes int
if s, err := fmt.Sscanf(step, "%dm", &minutes); s == 1 && err == nil {
stepSeconds = minutes * 60
}
}
}
// Build time condition
var timeCondition string
if start != "" && end != "" {
timeCondition = fmt.Sprintf("WHERE timestamp BETWEEN datetime(%s, 'unixepoch') AND datetime(%s, 'unixepoch')", start, end)
} else {
timeCondition = "WHERE timestamp >= datetime('now', '-24 hours')"
}
// Determine what metric to query
var sqlQuery string
var metricName string
switch query {
case "http_requests_total":
metricName = "http_requests_total"
sqlQuery = fmt.Sprintf(`
SELECT
strftime('%%s', strftime('%%Y-%%m-%%d %%H:%%M:00', timestamp)) as ts,
COUNT(*) as value
FROM user_visits
%s
GROUP BY strftime('%%Y-%%m-%%d %%H:%%M:00', timestamp)
ORDER BY ts
`, timeCondition)
case "http_unique_users_total":
metricName = "http_unique_users_total"
sqlQuery = fmt.Sprintf(`
SELECT
strftime('%%s', strftime('%%Y-%%m-%%d %%H:%%M:00', timestamp)) as ts,
COUNT(DISTINCT user_hash) as value
FROM user_visits
%s
GROUP BY strftime('%%Y-%%m-%%d %%H:%%M:00', timestamp)
ORDER BY ts
`, timeCondition)
default:
return c.JSON(http.StatusOK, map[string]interface{}{
"status": "success",
"data": map[string]interface{}{
"resultType": "matrix",
"result": []interface{}{},
},
})
}
rows, err := ts.db.Query(sqlQuery)
if err != nil {
return c.JSON(http.StatusInternalServerError, map[string]string{
"error": "Failed to retrieve endpoint stats",
return c.JSON(http.StatusInternalServerError, map[string]interface{}{
"status": "error",
"error": "Failed to execute query",
})
}
defer rows.Close()
type PeriodStats struct {
Endpoint string `json:"endpoint"`
Method string `json:"method"`
HitCount int `json:"hit_count"`
UniqueUsers int `json:"unique_users"`
LastAccess time.Time `json:"last_access"`
FirstAccess time.Time `json:"first_access"`
}
var stats []PeriodStats
var values [][]interface{}
for rows.Next() {
var stat PeriodStats
err := rows.Scan(&stat.Endpoint, &stat.Method, &stat.HitCount,
&stat.UniqueUsers, &stat.LastAccess, &stat.FirstAccess)
var timestamp int64
var value int
err := rows.Scan(&timestamp, &value)
if err != nil {
continue
}
stats = append(stats, stat)
values = append(values, []interface{}{timestamp, fmt.Sprintf("%d", value)})
}
return c.JSON(http.StatusOK, map[string]interface{}{
"stats": stats,
"period": period,
"total_endpoints": len(stats),
"status": "success",
"data": map[string]interface{}{
"resultType": "matrix",
"result": []map[string]interface{}{
{
"metric": map[string]string{
"__name__": metricName,
},
"values": values,
},
},
},
})
}
// GetRecentVisits returns recent visits with optional filtering
func (ts *TrackingService) GetRecentVisits(c echo.Context) error {
limit := c.QueryParam("limit")
if limit == "" {
limit = "50"
}
// Grafana-compatible endpoints below
query := `
SELECT user_hash, endpoint, method, ip_address, user_agent, referer, timestamp
FROM user_visits
ORDER BY timestamp DESC
LIMIT ?
`
rows, err := ts.db.Query(query, limit)
if err != nil {
return c.JSON(http.StatusInternalServerError, map[string]string{
"error": "Failed to retrieve recent visits",
})
}
defer rows.Close()
var visits []UserVisit
for rows.Next() {
var visit UserVisit
err := rows.Scan(&visit.UserHash, &visit.Endpoint, &visit.Method,
&visit.IPAddress, &visit.UserAgent, &visit.Referer, &visit.Timestamp)
if err != nil {
continue
}
visits = append(visits, visit)
}
return c.JSON(http.StatusOK, map[string]interface{}{
"visits": visits,
"count": len(visits),
})
}
// GetTrackingSummary returns overall tracking summary with period support
func (ts *TrackingService) GetTrackingSummary(c echo.Context) error {
// GetMetrics returns Prometheus-style metrics for Grafana
func (ts *TrackingService) GetMetrics(c echo.Context) error {
period := c.QueryParam("period")
if period == "" {
period = "all"
period = "24h"
}
var timeCondition string
switch period {
case "day":
case "1h":
timeCondition = "WHERE timestamp >= datetime('now', '-1 hour')"
case "24h":
timeCondition = "WHERE timestamp >= datetime('now', '-1 day')"
case "week":
case "7d":
timeCondition = "WHERE timestamp >= datetime('now', '-7 days')"
case "month":
timeCondition = "WHERE timestamp >= datetime('now', '-1 month')"
case "year":
timeCondition = "WHERE timestamp >= datetime('now', '-1 year')"
case "30d":
timeCondition = "WHERE timestamp >= datetime('now', '-30 days')"
default:
timeCondition = ""
timeCondition = "WHERE timestamp >= datetime('now', '-1 day')"
}
var totalHits, totalUniqueUsers int
var oldestVisit, newestVisit time.Time
// Get total hits for the period
// Get basic metrics
var totalHits, uniqueUsers, uniqueEndpoints int
query := fmt.Sprintf("SELECT COUNT(*) FROM user_visits %s", timeCondition)
err := ts.db.QueryRow(query).Scan(&totalHits)
if err != nil {
return c.JSON(http.StatusInternalServerError, map[string]string{
"error": "Failed to get summary stats",
})
}
ts.db.QueryRow(query).Scan(&totalHits)
// Get unique users for the period
query = fmt.Sprintf("SELECT COUNT(DISTINCT user_hash) FROM user_visits %s", timeCondition)
err = ts.db.QueryRow(query).Scan(&totalUniqueUsers)
if err != nil {
totalUniqueUsers = 0
}
ts.db.QueryRow(query).Scan(&uniqueUsers)
// Get oldest and newest visits for the period
query = fmt.Sprintf("SELECT MIN(timestamp), MAX(timestamp) FROM user_visits %s", timeCondition)
ts.db.QueryRow(query).Scan(&oldestVisit, &newestVisit)
query = fmt.Sprintf("SELECT COUNT(DISTINCT endpoint) FROM user_visits %s", timeCondition)
ts.db.QueryRow(query).Scan(&uniqueEndpoints)
// Get unique endpoints for the period
query = fmt.Sprintf("SELECT COUNT(DISTINCT endpoint || method) FROM user_visits %s", timeCondition)
var totalEndpoints int
ts.db.QueryRow(query).Scan(&totalEndpoints)
// Return Prometheus-style metrics
response := fmt.Sprintf(`# HELP http_requests_total Total HTTP requests
# TYPE http_requests_total counter
http_requests_total %d
summary := map[string]interface{}{
"period": period,
"total_hits": totalHits,
"total_endpoints": totalEndpoints,
"total_unique_users": totalUniqueUsers,
"oldest_visit": oldestVisit,
"newest_visit": newestVisit,
"tracking_active": true,
}
# HELP http_unique_users_total Unique users
# TYPE http_unique_users_total gauge
http_unique_users_total %d
return c.JSON(http.StatusOK, summary)
# HELP http_unique_endpoints_total Unique endpoints accessed
# TYPE http_unique_endpoints_total gauge
http_unique_endpoints_total %d
`, totalHits, uniqueUsers, uniqueEndpoints)
return c.String(http.StatusOK, response)
}
// GetTimeTrends returns visit trends over time
func (ts *TrackingService) GetTimeTrends(c echo.Context) error {
granularity := c.QueryParam("granularity") // hour, day, week, month
period := c.QueryParam("period") // last 24h, 7d, 30d, 1y
// GetTimeSeriesData returns time-series data for Grafana charts
func (ts *TrackingService) GetTimeSeriesData(c echo.Context) error {
target := c.QueryParam("target") // What to measure: hits, users, endpoints
from := c.QueryParam("from") // Start time (unix timestamp or relative)
to := c.QueryParam("to") // End time
interval := c.QueryParam("interval") // Time interval: 1m, 5m, 1h, 1d
if granularity == "" {
granularity = "day"
if target == "" {
target = "hits"
}
if period == "" {
period = "7d"
if interval == "" {
interval = "1h"
}
var timeFormat, timeCondition string
// Set time formatting based on granularity
switch granularity {
case "hour":
// Convert interval to SQLite format
var timeFormat string
switch interval {
case "1m":
timeFormat = "strftime('%Y-%m-%d %H:%M', timestamp)"
case "5m":
timeFormat = "strftime('%Y-%m-%d %H:%M', datetime(timestamp, 'unixepoch', 'start of minute', '+' || (strftime('%M', timestamp) / 5) * 5 || ' minutes'))"
case "1h":
timeFormat = "strftime('%Y-%m-%d %H:00', timestamp)"
case "day":
case "1d":
timeFormat = "strftime('%Y-%m-%d', timestamp)"
case "week":
timeFormat = "strftime('%Y-W%W', timestamp)"
case "month":
timeFormat = "strftime('%Y-%m', timestamp)"
default:
timeFormat = "strftime('%Y-%m-%d', timestamp)"
timeFormat = "strftime('%Y-%m-%d %H:00', timestamp)"
}
// Set time condition based on period
switch period {
case "24h":
timeCondition = "timestamp >= datetime('now', '-1 day')"
case "7d":
timeCondition = "timestamp >= datetime('now', '-7 days')"
case "30d":
timeCondition = "timestamp >= datetime('now', '-30 days')"
case "1y":
timeCondition = "timestamp >= datetime('now', '-1 year')"
// Build time condition
var timeCondition string
if from != "" && to != "" {
timeCondition = fmt.Sprintf("WHERE timestamp BETWEEN datetime('%s') AND datetime('%s')", from, to)
} else {
timeCondition = "WHERE timestamp >= datetime('now', '-24 hours')"
}
// Build query based on target
var selectClause string
switch target {
case "hits":
selectClause = "COUNT(*) as value"
case "users":
selectClause = "COUNT(DISTINCT user_hash) as value"
case "endpoints":
selectClause = "COUNT(DISTINCT endpoint) as value"
default:
timeCondition = "timestamp >= datetime('now', '-7 days')"
selectClause = "COUNT(*) as value"
}
query := fmt.Sprintf(`
SELECT
%s as time_bucket,
COUNT(*) as hits,
COUNT(DISTINCT user_hash) as unique_users,
COUNT(DISTINCT endpoint) as unique_endpoints
%s,
strftime('%%s', %s) * 1000 as timestamp_ms
FROM user_visits
WHERE %s
%s
GROUP BY time_bucket
ORDER BY time_bucket DESC
`, timeFormat, timeCondition)
ORDER BY timestamp_ms ASC
`, timeFormat, selectClause, timeFormat, timeCondition)
rows, err := ts.db.Query(query)
if err != nil {
return c.JSON(http.StatusInternalServerError, map[string]string{
"error": "Failed to retrieve time trends",
"error": "Failed to retrieve time series data",
})
}
defer rows.Close()
type TrendPoint struct {
TimeBucket string `json:"time_bucket"`
Hits int `json:"hits"`
UniqueUsers int `json:"unique_users"`
UniqueEndpoints int `json:"unique_endpoints"`
type DataPoint struct {
Target string `json:"target"`
DataPoints [][2]interface{} `json:"datapoints"` // [value, timestamp_ms]
}
var trends []TrendPoint
var dataPoints [][2]interface{}
for rows.Next() {
var trend TrendPoint
err := rows.Scan(&trend.TimeBucket, &trend.Hits, &trend.UniqueUsers, &trend.UniqueEndpoints)
var timeBucket string
var value int
var timestampMs int64
err := rows.Scan(&timeBucket, &value, &timestampMs)
if err != nil {
continue
}
trends = append(trends, trend)
dataPoints = append(dataPoints, [2]interface{}{value, timestampMs})
}
return c.JSON(http.StatusOK, map[string]interface{}{
"trends": trends,
"granularity": granularity,
"period": period,
})
response := []DataPoint{
{
Target: target,
DataPoints: dataPoints,
},
}
return c.JSON(http.StatusOK, response)
}
func (ts *TrackingService) Close() error {
return ts.db.Close()
// GetGrafanaQuery handles Grafana's query format
func (ts *TrackingService) GetGrafanaQuery(c echo.Context) error {
// This endpoint mimics Grafana's SimpleJSON datasource format
var request struct {
PanelId int `json:"panelId"`
Range struct {
From string `json:"from"`
To string `json:"to"`
} `json:"range"`
RangeRaw struct {
From string `json:"from"`
To string `json:"to"`
} `json:"rangeRaw"`
Interval string `json:"interval"`
IntervalMs int `json:"intervalMs"`
Targets []struct {
Target string `json:"target"`
RefId string `json:"refId"`
Type string `json:"type"`
} `json:"targets"`
Format string `json:"format"`
MaxDataPoints int `json:"maxDataPoints"`
}
if err := c.Bind(&request); err != nil {
return c.JSON(http.StatusBadRequest, map[string]string{
"error": "Invalid request format",
})
}
var results []interface{}
for _, target := range request.Targets {
// Parse the target to determine what to query
// Format: "metric_name.endpoint_filter.method_filter"
parts := strings.Split(target.Target, ".")
metricName := parts[0]
var timeCondition string
if request.Range.From != "" && request.Range.To != "" {
timeCondition = fmt.Sprintf("WHERE timestamp BETWEEN '%s' AND '%s'",
request.Range.From, request.Range.To)
} else {
timeCondition = "WHERE timestamp >= datetime('now', '-24 hours')"
}
switch metricName {
case "http_requests":
results = append(results, ts.getRequestsTimeSeries(timeCondition, target.RefId))
case "unique_users":
results = append(results, ts.getUniqueUsersTimeSeries(timeCondition, target.RefId))
case "top_endpoints":
results = append(results, ts.getTopEndpoints(timeCondition, target.RefId))
}
}
return c.JSON(http.StatusOK, results)
}
// Helper functions for Grafana queries
func (ts *TrackingService) getRequestsTimeSeries(timeCondition, refId string) map[string]interface{} {
query := fmt.Sprintf(`
SELECT
strftime('%%Y-%%m-%%d %%H:00', timestamp) as hour,
COUNT(*) as requests,
strftime('%%s', strftime('%%Y-%%m-%%d %%H:00', timestamp)) * 1000 as timestamp_ms
FROM user_visits
%s
GROUP BY hour
ORDER BY timestamp_ms ASC
`, timeCondition)
rows, _ := ts.db.Query(query)
defer rows.Close()
var dataPoints [][2]interface{}
for rows.Next() {
var hour string
var requests int
var timestampMs int64
rows.Scan(&hour, &requests, &timestampMs)
dataPoints = append(dataPoints, [2]interface{}{requests, timestampMs})
}
return map[string]interface{}{
"target": "HTTP Requests",
"refId": refId,
"datapoints": dataPoints,
}
}
func (ts *TrackingService) getUniqueUsersTimeSeries(timeCondition, refId string) map[string]interface{} {
query := fmt.Sprintf(`
SELECT
strftime('%%Y-%%m-%%d %%H:00', timestamp) as hour,
COUNT(DISTINCT user_hash) as users,
strftime('%%s', strftime('%%Y-%%m-%%d %%H:00', timestamp)) * 1000 as timestamp_ms
FROM user_visits
%s
GROUP BY hour
ORDER BY timestamp_ms ASC
`, timeCondition)
rows, _ := ts.db.Query(query)
defer rows.Close()
var dataPoints [][2]interface{}
for rows.Next() {
var hour string
var users int
var timestampMs int64
rows.Scan(&hour, &users, &timestampMs)
dataPoints = append(dataPoints, [2]interface{}{users, timestampMs})
}
return map[string]interface{}{
"target": "Unique Users",
"refId": refId,
"datapoints": dataPoints,
}
}
func (ts *TrackingService) getTopEndpoints(timeCondition, refId string) map[string]interface{} {
query := fmt.Sprintf(`
SELECT
endpoint,
COUNT(*) as hits
FROM user_visits
%s
GROUP BY endpoint
ORDER BY hits DESC
LIMIT 10
`, timeCondition)
rows, _ := ts.db.Query(query)
defer rows.Close()
type EndpointData struct {
Endpoint string `json:"endpoint"`
Hits int `json:"hits"`
}
var endpoints []EndpointData
for rows.Next() {
var endpoint string
var hits int
rows.Scan(&endpoint, &hits)
endpoints = append(endpoints, EndpointData{Endpoint: endpoint, Hits: hits})
}
return map[string]interface{}{
"target": "Top Endpoints",
"refId": refId,
"type": "table",
"columns": []map[string]string{
{"text": "Endpoint", "type": "string"},
{"text": "Hits", "type": "number"},
},
"rows": endpoints,
}
}

View File

@@ -5,11 +5,7 @@ import (
"encoding/base64"
"fmt"
"io"
"net/http"
"os"
"strings"
"github.com/labstack/echo/v4"
)
func GetBuildNumber() string {
@@ -68,37 +64,3 @@ func CalculateAssetIntegrities() *AssetIntegrity {
return integrity
}
// BearerAuthMiddleware provides simple bearer token authentication
func BearerAuthMiddleware() echo.MiddlewareFunc {
return func(next echo.HandlerFunc) echo.HandlerFunc {
return func(c echo.Context) error {
// Get the authorization header
auth := c.Request().Header.Get("Authorization")
if auth == "" {
return echo.NewHTTPError(http.StatusUnauthorized, "Missing authorization header")
}
// Check if it starts with "Bearer "
if !strings.HasPrefix(auth, "Bearer ") {
return echo.NewHTTPError(http.StatusUnauthorized, "Invalid authorization format")
}
// Extract the token
token := strings.TrimPrefix(auth, "Bearer ")
// Get expected token from environment
expectedToken := os.Getenv("API_TOKEN")
if expectedToken == "" {
return echo.NewHTTPError(http.StatusInternalServerError, "Server configuration error")
}
// Compare tokens
if token != expectedToken {
return echo.NewHTTPError(http.StatusUnauthorized, "Invalid token")
}
return next(c)
}
}
}