From 5e162b4be54c939754da64a18bb10a44838535a1 Mon Sep 17 00:00:00 2001 From: "Alexander \"PapaTutuWawa" Date: Fri, 9 Feb 2024 00:06:43 +0100 Subject: [PATCH] Initial commit --- .gitignore | 1 + go.mod | 24 +++++++ go.sum | 52 ++++++++++++++ main.go | 208 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 285 insertions(+) create mode 100644 .gitignore create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..88d050b --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +main \ No newline at end of file diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..3743b63 --- /dev/null +++ b/go.mod @@ -0,0 +1,24 @@ +module polynom.me/polynom.me/page-metrics + +go 1.21.5 + +require ( + github.com/go-co-op/gocron/v2 v2.2.4 + github.com/prometheus/client_golang v1.18.0 + github.com/sirupsen/logrus v1.9.3 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/jonboulle/clockwork v0.4.0 // indirect + github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.45.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect + github.com/robfig/cron/v3 v3.0.1 // indirect + golang.org/x/exp v0.0.0-20231219180239-dc181d75b848 // indirect + golang.org/x/sys v0.15.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..e9ca786 --- /dev/null +++ b/go.sum @@ -0,0 +1,52 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-co-op/gocron/v2 v2.2.4 h1:fL6a8/U+BJQ9UbaeqKxua8wY02w4ftKZsxPzLSNOCKk= +github.com/go-co-op/gocron/v2 v2.2.4/go.mod h1:igssOwzZkfcnu3m2kwnCf/mYj4SmhP9ecSgmYjCOHkk= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= +github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= +github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= +github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +golang.org/x/exp v0.0.0-20231219180239-dc181d75b848 h1:+iq7lrkxmFNBM7xx+Rae2W6uyPfhPeDWD+n+JgppptE= +golang.org/x/exp v0.0.0-20231219180239-dc181d75b848/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go new file mode 100644 index 0000000..dcc94c3 --- /dev/null +++ b/main.go @@ -0,0 +1,208 @@ +package main + +import ( + "encoding/json" + "io" + "net/http" + "sync" + + "github.com/go-co-op/gocron/v2" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + + log "github.com/sirupsen/logrus" +) + +type DomainState struct { + // Hits per page (file) + Pages map[string]int + PagesMetrics map[string]prometheus.Gauge + + // Referrals + Referrals map[string]int + ReferralsMetrics map[string]prometheus.Gauge +} + +type State struct { + // Maps domain names to the sliding window + Domains map[string]DomainState + + // Lock guarding the access to Domains + Lock sync.Mutex +} + +func increment(key string, dict *map[string]int) { + v, found := (*dict)[key] + if !found { + (*dict)[key] = 1 + } else { + (*dict)[key] = v + 1 + } +} + +type Track struct { + Domain string `json:"domain"` + Referer string `json:"referer"` + Path string `json:"path"` +} + +func ingestHandler(registry *prometheus.Registry, state *State) http.HandlerFunc { + return func(w http.ResponseWriter, req *http.Request) { + if req.Method != http.MethodPost { + log.Debugf("Unexpected %s request", req.Method) + w.WriteHeader(http.StatusBadRequest) + return + } + + body, err := io.ReadAll(req.Body) + if err != nil { + log.Debugf("Failed to read body: %v", err) + w.WriteHeader(http.StatusBadRequest) + return + } + + var track Track + err = json.Unmarshal(body, &track) + if err != nil { + log.Debugf("Failed to unmarshal request body: %v", err) + w.WriteHeader(http.StatusBadRequest) + return + } + + // Acquire lock + state.Lock.Lock() + defer state.Lock.Unlock() + + // Update everything + log.Debugf("%v", state.Domains) + log.Debugf("Got Domain: %s", track.Domain) + domainState, found := state.Domains[track.Domain] + if !found { + log.Debugf("Domain %s not found. Creating state", track.Domain) + domainState = DomainState{ + Pages: make(map[string]int), + PagesMetrics: make(map[string]prometheus.Gauge), + Referrals: make(map[string]int), + ReferralsMetrics: make(map[string]prometheus.Gauge), + } + state.Domains[track.Domain] = domainState + } + + // Record the referrer + referrer := track.Referer + if referrer != "" { + v, found := domainState.Referrals[referrer] + if found { + value := v + 1 + domainState.Referrals[referrer] = value + + metric, _ := domainState.ReferralsMetrics[referrer] + metric.Set(float64(value)) + } else { + metric := prometheus.NewGauge( + prometheus.GaugeOpts{ + Name: "referals", + ConstLabels: prometheus.Labels{"domain": track.Domain, "referer": referrer}, + }, + ) + metric.Set(1) + domainState.Referrals[referrer] = 1 + domainState.ReferralsMetrics[referrer] = metric + log.Debugf("Registering gauge \"referals\" with referer=%s", referrer) + registry.MustRegister(metric) + } + } + + // Record the file + path := track.Path + v, found := domainState.Pages[path] + log.Debugf("v=%d, found=%v", v, found) + if found { + value := v + 1 + domainState.Pages[path] = value + + metric, _ := domainState.PagesMetrics[path] + metric.Set(float64(value)) + } else { + metric := prometheus.NewGauge( + prometheus.GaugeOpts{ + Name: "pages", + ConstLabels: prometheus.Labels{"domain": track.Domain, "path": path}, + }, + ) + metric.Set(1) + domainState.Pages[path] = 1 + domainState.PagesMetrics[path] = metric + log.Debugf("Registering gauge \"path\" with path=%s", path) + registry.MustRegister(metric) + } + state.Domains[track.Domain] = domainState + + w.WriteHeader(http.StatusAccepted) + } +} + +func main() { + log.SetLevel(log.DebugLevel) + + // Setup metrics + registry := prometheus.NewRegistry() + state := State{ + Domains: make(map[string]DomainState), + } + + // Setup the scheduler + scheduler, err := gocron.NewScheduler() + if err != nil { + log.Fatalf("Failed to create scheduler: %v", err) + return + } + _, err = scheduler.NewJob( + gocron.DailyJob(1, gocron.NewAtTimes(gocron.NewAtTime(0, 0, 0))), + gocron.NewTask( + func() { + // Acquire the lock + log.Info("Starting reset procedure...") + state.Lock.Lock() + defer state.Lock.Unlock() + + for domainName, domain := range state.Domains { + log.Infof("Resetting counters for %s", domainName) + + // Reset pages + for page := range domain.Pages { + domain.Pages[page] = 0 + domain.PagesMetrics[page].Set(0) + } + + // Reset referals + for referer := range domain.Referrals { + domain.Referrals[referer] = 0 + domain.ReferralsMetrics[referer].Set(0) + } + } + }, + ), + ) + if err != nil { + log.Fatalf("Failed to create task: %v", err) + return + } + scheduler.Start() + + // Set up the HTTP server + http.Handle( + "/metrics", promhttp.HandlerFor( + registry, + promhttp.HandlerOpts{ + EnableOpenMetrics: true, + }, + ), + ) + http.Handle( + "/track", ingestHandler(registry, &state), + ) + addr := "127.0.0.1:9999" + log.Infof("Starting server at %s", addr) + http.ListenAndServe(addr, nil) +}