go-judge/cmd/executorserver/metrics.go
criyle 78bb4d85de filestore: use /dev/shm instead of memory as file storage
- increase default tmpfs size to 128M
- add /config to get file store path
- remove memory only file store

close #20
2021-09-19 15:47:24 -07:00

224 lines
5.3 KiB
Go

package main
import (
"os"
"sync"
"time"
"github.com/criyle/go-judge/env/pool"
"github.com/criyle/go-judge/envexec"
"github.com/criyle/go-judge/filestore"
"github.com/criyle/go-judge/worker"
"github.com/prometheus/client_golang/prometheus"
)
const (
metricsNamespace = "executorserver"
)
var (
// 1ms -> 10s
timeBuckets = []float64{
0.001, 0.002, 0.005, 0.008, 0.010, 0.025, 0.050, 0.075, 0.1, 0.2,
0.4, 0.6, 0.8, 1.0, 1.5, 2, 5, 10,
}
// 4k (1<<12) -> 4g (1<<32)
memoryBucket = prometheus.ExponentialBuckets(1<<12, 2, 21)
// 256 byte (1<<8) -> 256m (1<<28)
fileSizeBucket = prometheus.ExponentialBuckets(1<<8, 2, 20)
metricsSummaryQuantile = map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}
execErrorCount = prometheus.NewCounter(prometheus.CounterOpts{
Namespace: metricsNamespace,
Name: "error",
Help: "Number of exec query returns error",
})
execTimeHist = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: metricsNamespace,
Name: "time_seconds",
Help: "Histogram for the running time",
Buckets: timeBuckets,
}, []string{"status"})
execTimeSummary = prometheus.NewSummaryVec(prometheus.SummaryOpts{
Namespace: metricsNamespace,
Name: "time",
Help: "Summary for the running time",
Objectives: metricsSummaryQuantile,
}, []string{"status"})
execMemHist = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: metricsNamespace,
Name: "memory_bytes",
Help: "Histgram for the memory",
Buckets: memoryBucket,
}, []string{"status"})
execMemSummary = prometheus.NewSummaryVec(prometheus.SummaryOpts{
Namespace: metricsNamespace,
Name: "memory",
Help: "Summary for the memory",
Objectives: metricsSummaryQuantile,
}, []string{"status"})
fsSizeHist = prometheus.NewHistogram(prometheus.HistogramOpts{
Namespace: metricsNamespace,
Name: "file_size_bytes",
Help: "Histgram for the file size in the file store",
Buckets: fileSizeBucket,
})
fsSizeSummary = prometheus.NewSummary(prometheus.SummaryOpts{
Namespace: metricsNamespace,
Name: "file_size",
Help: "Summary for the file size in the file store",
Objectives: metricsSummaryQuantile,
})
fsTotalCount = prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: metricsNamespace,
Name: "file_current_total",
Help: "Total number of current files in the file store",
})
fsTotalSize = prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: metricsNamespace,
Name: "file_size_current_total",
Help: "Total size of current files in the file store",
})
envCreated = prometheus.NewCounter(prometheus.CounterOpts{
Namespace: metricsNamespace,
Name: "environment_created",
Help: "Total number of environment build by environment builder",
})
envInUse = prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: metricsNamespace,
Name: "environment_in_use",
Help: "Total number of environment currently in use",
})
)
func init() {
prometheus.MustRegister(execErrorCount)
prometheus.MustRegister(execTimeHist, execTimeSummary)
prometheus.MustRegister(execMemHist, execMemSummary)
prometheus.MustRegister(fsSizeHist, fsSizeSummary, fsTotalSize)
prometheus.MustRegister(envCreated, envInUse)
}
func execObserve(res worker.Response) {
if res.Error != nil {
execErrorCount.Inc()
}
for _, r := range res.Results {
status := r.Status.String()
d := time.Duration(r.Time)
ob := d.Seconds()
mob := float64(r.Memory)
execTimeHist.WithLabelValues(status).Observe(ob)
execTimeSummary.WithLabelValues(status).Observe(ob)
execMemHist.WithLabelValues(status).Observe(mob)
execMemSummary.WithLabelValues(status).Observe(mob)
}
}
var _ filestore.FileStore = &metricsFileStore{}
type metricsFileStore struct {
mu sync.Mutex
filestore.FileStore
fileSize map[string]int64
}
func newMetricsFileStore(fs filestore.FileStore) filestore.FileStore {
return &metricsFileStore{
FileStore: fs,
fileSize: make(map[string]int64),
}
}
func (m *metricsFileStore) Add(name, path string) (string, error) {
id, err := m.FileStore.Add(name, path)
if err != nil {
return "", err
}
m.mu.Lock()
defer m.mu.Unlock()
fi, err := os.Stat(path)
if err != nil {
return id, nil
}
s := fi.Size()
m.fileSize[id] = s
sf := float64(s)
fsSizeHist.Observe(sf)
fsSizeSummary.Observe(sf)
fsTotalSize.Add(sf)
fsTotalCount.Inc()
return id, nil
}
func (m *metricsFileStore) Remove(id string) bool {
success := m.FileStore.Remove(id)
m.mu.Lock()
defer m.mu.Unlock()
s, ok := m.fileSize[id]
if !ok {
return success
}
delete(m.fileSize, id)
sf := float64(s)
fsTotalSize.Sub(sf)
fsTotalCount.Dec()
return success
}
var _ pool.EnvBuilder = &metriceEnvBuilder{}
type metriceEnvBuilder struct {
pool.EnvBuilder
}
func (b *metriceEnvBuilder) Build() (pool.Environment, error) {
e, err := b.EnvBuilder.Build()
if err != nil {
return nil, err
}
envCreated.Inc()
return e, nil
}
var _ worker.EnvironmentPool = &metricsEnvPool{}
type metricsEnvPool struct {
worker.EnvironmentPool
}
func (p *metricsEnvPool) Get() (envexec.Environment, error) {
e, err := p.EnvironmentPool.Get()
if err != nil {
return nil, err
}
envInUse.Inc()
return e, nil
}
func (p *metricsEnvPool) Put(env envexec.Environment) {
p.EnvironmentPool.Put(env)
envInUse.Dec()
}