git init
This commit is contained in:
4
control-plane/backend/.dockerignore
Normal file
4
control-plane/backend/.dockerignore
Normal file
@@ -0,0 +1,4 @@
|
||||
backend
|
||||
*_test.go
|
||||
.git
|
||||
|
||||
17
control-plane/backend/Dockerfile
Normal file
17
control-plane/backend/Dockerfile
Normal file
@@ -0,0 +1,17 @@
|
||||
# syntax=docker/dockerfile:1.7
|
||||
FROM golang:1.24-alpine AS build
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN --mount=type=cache,target=/go/pkg/mod go mod download
|
||||
COPY . .
|
||||
RUN --mount=type=cache,target=/root/.cache/go-build \
|
||||
CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/control-plane .
|
||||
|
||||
FROM alpine:3.22
|
||||
RUN apk add --no-cache ca-certificates tzdata && \
|
||||
addgroup -S -g 65532 control && adduser -S -D -H -u 65532 -G control control
|
||||
COPY --from=build --chown=65532:65532 /out/control-plane /usr/local/bin/control-plane
|
||||
USER 65532:65532
|
||||
EXPOSE 8080
|
||||
ENTRYPOINT ["/usr/local/bin/control-plane"]
|
||||
|
||||
149
control-plane/backend/agent.go
Normal file
149
control-plane/backend/agent.go
Normal file
@@ -0,0 +1,149 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const maxAgentResponse = 2 << 20
|
||||
|
||||
type AgentClient struct {
|
||||
http *http.Client
|
||||
credentialsDir string
|
||||
}
|
||||
|
||||
func NewAgentClient(credentialsDir string) *AgentClient {
|
||||
transport := &http.Transport{
|
||||
Proxy: nil,
|
||||
DialContext: (&net.Dialer{Timeout: 3 * time.Second, KeepAlive: 30 * time.Second}).DialContext,
|
||||
TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS12},
|
||||
TLSHandshakeTimeout: 3 * time.Second,
|
||||
ResponseHeaderTimeout: 5 * time.Second,
|
||||
MaxConnsPerHost: 8,
|
||||
MaxIdleConnsPerHost: 4,
|
||||
IdleConnTimeout: 30 * time.Second,
|
||||
}
|
||||
return &AgentClient{
|
||||
http: &http.Client{
|
||||
Transport: transport,
|
||||
Timeout: 7 * time.Second,
|
||||
CheckRedirect: func(_ *http.Request, _ []*http.Request) error {
|
||||
return errors.New("agent redirects are disabled")
|
||||
},
|
||||
},
|
||||
credentialsDir: credentialsDir,
|
||||
}
|
||||
}
|
||||
|
||||
func validateAgentEndpoint(endpoint string) error {
|
||||
parsed, err := url.Parse(endpoint)
|
||||
if err != nil {
|
||||
return errors.New("invalid URL")
|
||||
}
|
||||
if parsed.Scheme != "http" && parsed.Scheme != "https" {
|
||||
return errors.New("scheme must be http or https")
|
||||
}
|
||||
if parsed.User != nil || (parsed.Path != "" && parsed.Path != "/") || parsed.RawQuery != "" || parsed.Fragment != "" {
|
||||
return errors.New("endpoint must contain only scheme, private IP, and port")
|
||||
}
|
||||
if parsed.Port() == "" {
|
||||
return errors.New("endpoint must include a port")
|
||||
}
|
||||
ip := net.ParseIP(parsed.Hostname())
|
||||
if ip == nil || !(ip.IsPrivate() || ip.IsLoopback() || ip.IsLinkLocalUnicast()) {
|
||||
return errors.New("endpoint must use a private literal IP address")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validCredentialRef(reference string) bool {
|
||||
if len(reference) < 1 || len(reference) > 128 {
|
||||
return false
|
||||
}
|
||||
for _, char := range reference {
|
||||
if !(char >= 'a' && char <= 'z') && !(char >= 'A' && char <= 'Z') &&
|
||||
!(char >= '0' && char <= '9') && char != '_' && char != '-' && char != '.' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return reference != "." && reference != ".."
|
||||
}
|
||||
|
||||
func (client *AgentClient) credential(reference string) (string, error) {
|
||||
if !validCredentialRef(reference) {
|
||||
return "", errors.New("invalid credential reference")
|
||||
}
|
||||
path := filepath.Join(client.credentialsDir, reference)
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
||||
return "", errors.New("credential must be a regular file")
|
||||
}
|
||||
// Docker Compose file-backed secrets are mounted read-only as 0444 on some
|
||||
// engines. Accept read-only mounts, but never a group/world-writable secret.
|
||||
if info.Mode().Perm()&0o022 != 0 {
|
||||
return "", errors.New("credential file must not be group/world writable")
|
||||
}
|
||||
if info.Size() > 4096 {
|
||||
return "", errors.New("credential file is too large")
|
||||
}
|
||||
value, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
token := strings.TrimSpace(string(value))
|
||||
if len(token) < 32 {
|
||||
return "", errors.New("credential is too short")
|
||||
}
|
||||
return token, nil
|
||||
}
|
||||
|
||||
func (client *AgentClient) Do(ctx context.Context, server Server, method, path string, body []byte) (int, []byte, error) {
|
||||
if err := validateAgentEndpoint(server.AgentEndpoint); err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
if !strings.HasPrefix(path, "/v1/") {
|
||||
return 0, nil, errors.New("invalid agent API path")
|
||||
}
|
||||
token, err := client.credential(server.CredentialRef)
|
||||
if err != nil {
|
||||
return 0, nil, fmt.Errorf("load agent credential: %w", err)
|
||||
}
|
||||
request, err := http.NewRequestWithContext(ctx, method,
|
||||
strings.TrimRight(server.AgentEndpoint, "/")+path, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
request.Header.Set("Authorization", "Bearer "+token)
|
||||
request.Header.Set("Accept", "application/json")
|
||||
if len(body) > 0 {
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
response, err := client.http.Do(request)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
limited := io.LimitReader(response.Body, maxAgentResponse+1)
|
||||
payload, err := io.ReadAll(limited)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
if len(payload) > maxAgentResponse {
|
||||
return 0, nil, errors.New("agent response is too large")
|
||||
}
|
||||
return response.StatusCode, payload, nil
|
||||
}
|
||||
64
control-plane/backend/agent_test.go
Normal file
64
control-plane/backend/agent_test.go
Normal file
@@ -0,0 +1,64 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestAgentEndpointValidation(t *testing.T) {
|
||||
for _, endpoint := range []string{"http://127.0.0.1:9105", "https://10.10.0.2:9105", "http://[fd00::1]:9105"} {
|
||||
if err := validateAgentEndpoint(endpoint); err != nil {
|
||||
t.Fatalf("%s: %v", endpoint, err)
|
||||
}
|
||||
}
|
||||
for _, endpoint := range []string{"http://0.0.0.0:9105", "http://8.8.8.8:9105", "https://agent.example:9105", "file:///etc/passwd", "http://127.0.0.1:9105/path"} {
|
||||
if validateAgentEndpoint(endpoint) == nil {
|
||||
t.Fatalf("unsafe endpoint accepted: %s", endpoint)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCredentialReferences(t *testing.T) {
|
||||
if !validCredentialRef("agent_primary.token") {
|
||||
t.Fatal("valid ref rejected")
|
||||
}
|
||||
for _, value := range []string{"", "../secret", "path/name", ".", "bad ref"} {
|
||||
if validCredentialRef(value) {
|
||||
t.Fatalf("invalid ref accepted: %q", value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentClientAuthenticatesAndBoundsPath(t *testing.T) {
|
||||
token := strings.Repeat("a", 32)
|
||||
dir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(dir, "agent"), []byte(token), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client := NewAgentClient(dir)
|
||||
client.http.Transport = roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||
if request.Header.Get("Authorization") != "Bearer "+token {
|
||||
t.Error("missing token")
|
||||
}
|
||||
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"ok":true}`))}, nil
|
||||
})
|
||||
server := Server{AgentEndpoint: "http://127.0.0.1:9105", CredentialRef: "agent"}
|
||||
status, _, err := client.Do(context.Background(), server, http.MethodGet, "/v1/version", nil)
|
||||
if err != nil || status != 200 {
|
||||
t.Fatalf("agent request failed: status=%d err=%v", status, err)
|
||||
}
|
||||
if _, _, err = client.Do(context.Background(), server, http.MethodGet, "/private", nil); err == nil {
|
||||
t.Fatal("out-of-contract path accepted")
|
||||
}
|
||||
}
|
||||
|
||||
type roundTripFunc func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (function roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) {
|
||||
return function(request)
|
||||
}
|
||||
459
control-plane/backend/api.go
Normal file
459
control-plane/backend/api.go
Normal file
@@ -0,0 +1,459 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/subtle"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
const maxRequestBody = 64 << 10
|
||||
|
||||
type App struct {
|
||||
store *Store
|
||||
agent *AgentClient
|
||||
config Config
|
||||
logins *LoginLimiter
|
||||
integrationLimiter *LoginLimiter
|
||||
integrationTokenHash []byte
|
||||
dummyHash string
|
||||
}
|
||||
|
||||
func NewApp(store *Store, agent *AgentClient, config Config) (*App, error) {
|
||||
dummy, err := HashPassword("unused-password-for-timing-equalization")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var integrationTokenHash []byte
|
||||
if config.IntegrationTokenFile != "" {
|
||||
token, tokenErr := readIntegrationToken(config.IntegrationTokenFile)
|
||||
if tokenErr != nil {
|
||||
return nil, tokenErr
|
||||
}
|
||||
integrationTokenHash = tokenHash(token)
|
||||
}
|
||||
return &App{store: store, agent: agent, config: config,
|
||||
logins: NewLoginLimiter(5, time.Minute), integrationLimiter: NewLoginLimiter(600, time.Minute),
|
||||
integrationTokenHash: integrationTokenHash, dummyHash: dummy}, nil
|
||||
}
|
||||
|
||||
func (app *App) ServeHTTP(writer http.ResponseWriter, request *http.Request) {
|
||||
writer.Header().Set("X-Content-Type-Options", "nosniff")
|
||||
writer.Header().Set("X-Frame-Options", "DENY")
|
||||
writer.Header().Set("Referrer-Policy", "no-referrer")
|
||||
writer.Header().Set("Cache-Control", "no-store")
|
||||
writer.Header().Set("Content-Security-Policy", "default-src 'none'; frame-ancestors 'none'")
|
||||
if strings.HasPrefix(request.URL.Path, "/integrations/") {
|
||||
app.integration(writer, request)
|
||||
return
|
||||
}
|
||||
|
||||
if request.URL.Path == "/healthz" && request.Method == http.MethodGet {
|
||||
ctx, cancel := context.WithTimeout(request.Context(), 2*time.Second)
|
||||
defer cancel()
|
||||
if err := app.store.pool.Ping(ctx); err != nil {
|
||||
writeError(writer, http.StatusServiceUnavailable, "database unavailable", "unhealthy")
|
||||
return
|
||||
}
|
||||
writeJSON(writer, http.StatusOK, map[string]string{"status": "ok"})
|
||||
return
|
||||
}
|
||||
if request.URL.Path == "/api/v1/auth/login" && request.Method == http.MethodPost {
|
||||
app.login(writer, request)
|
||||
return
|
||||
}
|
||||
if !strings.HasPrefix(request.URL.Path, "/api/v1/") {
|
||||
writeError(writer, http.StatusNotFound, "not found", "not_found")
|
||||
return
|
||||
}
|
||||
user, session, csrfHash, ok := app.authenticate(writer, request)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
switch {
|
||||
case request.URL.Path == "/api/v1/auth/me" && request.Method == http.MethodGet:
|
||||
writeJSON(writer, http.StatusOK, user)
|
||||
case request.URL.Path == "/api/v1/auth/logout" && request.Method == http.MethodPost:
|
||||
if !app.requireCSRF(writer, request, csrfHash) {
|
||||
return
|
||||
}
|
||||
_ = app.store.DeleteSession(request.Context(), session)
|
||||
app.clearCookies(writer)
|
||||
app.store.Audit(request.Context(), &user, "", "auth.logout", user.Username, nil, nil, "success", app.clientIP(request))
|
||||
writer.WriteHeader(http.StatusNoContent)
|
||||
case request.URL.Path == "/api/v1/servers" && request.Method == http.MethodGet:
|
||||
app.listServers(writer, request)
|
||||
case request.URL.Path == "/api/v1/servers" && request.Method == http.MethodPost:
|
||||
if !app.authorize(writer, request, user, csrfHash, RoleAdministrator) {
|
||||
return
|
||||
}
|
||||
app.createServer(writer, request, user)
|
||||
case request.URL.Path == "/api/v1/audit" && request.Method == http.MethodGet:
|
||||
app.listAudit(writer, request)
|
||||
case strings.HasPrefix(request.URL.Path, "/api/v1/servers/"):
|
||||
app.serverRoute(writer, request, user, csrfHash)
|
||||
default:
|
||||
writeError(writer, http.StatusNotFound, "not found", "not_found")
|
||||
}
|
||||
}
|
||||
|
||||
func (app *App) login(writer http.ResponseWriter, request *http.Request) {
|
||||
ip := app.clientIP(request)
|
||||
if !app.logins.Allow(ip, time.Now()) {
|
||||
writeError(writer, http.StatusTooManyRequests, "too many login attempts", "rate_limited")
|
||||
return
|
||||
}
|
||||
var input struct {
|
||||
Username string `json:"username"`
|
||||
Password string `json:"password"`
|
||||
}
|
||||
if err := decodeJSON(writer, request, &input); err != nil {
|
||||
writeError(writer, http.StatusBadRequest, "invalid request", "invalid_request")
|
||||
return
|
||||
}
|
||||
user, hash, disabled, err := app.store.LoginUser(request.Context(), input.Username)
|
||||
if err != nil {
|
||||
log.Printf("login query: %v", err)
|
||||
writeError(writer, http.StatusInternalServerError, "internal error", "internal_error")
|
||||
return
|
||||
}
|
||||
if hash == "" {
|
||||
hash = app.dummyHash
|
||||
}
|
||||
valid := VerifyPassword(hash, input.Password) && user.ID != "" && !disabled
|
||||
if !valid {
|
||||
app.store.Audit(request.Context(), nil, "", "auth.login", input.Username, nil, nil, "failure", ip)
|
||||
writeError(writer, http.StatusUnauthorized, "invalid credentials", "invalid_credentials")
|
||||
return
|
||||
}
|
||||
session, csrf, err := app.store.CreateSession(request.Context(), user.ID, ip, app.config.SessionTTL)
|
||||
if err != nil {
|
||||
log.Printf("create session: %v", err)
|
||||
writeError(writer, http.StatusInternalServerError, "internal error", "internal_error")
|
||||
return
|
||||
}
|
||||
app.setCookies(writer, session, csrf)
|
||||
app.store.Audit(request.Context(), &user, "", "auth.login", user.Username, nil, nil, "success", ip)
|
||||
writeJSON(writer, http.StatusOK, user)
|
||||
}
|
||||
|
||||
func (app *App) authenticate(writer http.ResponseWriter, request *http.Request) (User, string, []byte, bool) {
|
||||
cookie, err := request.Cookie("vps_session")
|
||||
if err != nil || len(cookie.Value) > 256 {
|
||||
writeError(writer, http.StatusUnauthorized, "authentication required", "unauthorized")
|
||||
return User{}, "", nil, false
|
||||
}
|
||||
user, csrfHash, err := app.store.SessionUser(request.Context(), cookie.Value)
|
||||
if err != nil {
|
||||
if !errors.Is(err, pgx.ErrNoRows) {
|
||||
log.Printf("session query: %v", err)
|
||||
}
|
||||
writeError(writer, http.StatusUnauthorized, "authentication required", "unauthorized")
|
||||
return User{}, "", nil, false
|
||||
}
|
||||
return user, cookie.Value, csrfHash, true
|
||||
}
|
||||
|
||||
func (app *App) authorize(writer http.ResponseWriter, request *http.Request, user User, csrfHash []byte, role string) bool {
|
||||
if !permits(user.Role, role) {
|
||||
writeError(writer, http.StatusForbidden, "insufficient permissions", "forbidden")
|
||||
return false
|
||||
}
|
||||
return app.requireCSRF(writer, request, csrfHash)
|
||||
}
|
||||
|
||||
func (app *App) requireCSRF(writer http.ResponseWriter, request *http.Request, expected []byte) bool {
|
||||
provided := request.Header.Get("X-CSRF-Token")
|
||||
if len(provided) > 256 || subtle.ConstantTimeCompare(tokenHash(provided), expected) != 1 {
|
||||
writeError(writer, http.StatusForbidden, "CSRF validation failed", "csrf_failed")
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (app *App) setCookies(writer http.ResponseWriter, session, csrf string) {
|
||||
maxAge := int(app.config.SessionTTL.Seconds())
|
||||
http.SetCookie(writer, &http.Cookie{Name: "vps_session", Value: session, Path: "/", HttpOnly: true,
|
||||
Secure: app.config.CookieSecure, SameSite: http.SameSiteStrictMode, MaxAge: maxAge})
|
||||
http.SetCookie(writer, &http.Cookie{Name: "vps_csrf", Value: csrf, Path: "/", HttpOnly: false,
|
||||
Secure: app.config.CookieSecure, SameSite: http.SameSiteStrictMode, MaxAge: maxAge})
|
||||
}
|
||||
|
||||
func (app *App) clearCookies(writer http.ResponseWriter) {
|
||||
for _, name := range []string{"vps_session", "vps_csrf"} {
|
||||
http.SetCookie(writer, &http.Cookie{Name: name, Path: "/", HttpOnly: name == "vps_session",
|
||||
Secure: app.config.CookieSecure, SameSite: http.SameSiteStrictMode, MaxAge: -1})
|
||||
}
|
||||
}
|
||||
|
||||
func (app *App) listServers(writer http.ResponseWriter, request *http.Request) {
|
||||
servers, err := app.store.Servers(request.Context())
|
||||
if err != nil {
|
||||
writeInternal(writer, err)
|
||||
return
|
||||
}
|
||||
writeJSON(writer, http.StatusOK, servers)
|
||||
}
|
||||
|
||||
func (app *App) createServer(writer http.ResponseWriter, request *http.Request, user User) {
|
||||
var input struct {
|
||||
Name string `json:"name"`
|
||||
Hostname string `json:"hostname"`
|
||||
AgentEndpoint string `json:"agent_endpoint"`
|
||||
CredentialRef string `json:"credential_ref"`
|
||||
Tags []string `json:"tags"`
|
||||
}
|
||||
if err := decodeJSON(writer, request, &input); err != nil || len(input.Name) < 1 || len(input.Name) > 128 ||
|
||||
len(input.Hostname) > 255 || len(input.Tags) > 20 || validateAgentEndpoint(input.AgentEndpoint) != nil ||
|
||||
!validCredentialRef(input.CredentialRef) {
|
||||
writeError(writer, http.StatusBadRequest, "invalid server", "invalid_request")
|
||||
return
|
||||
}
|
||||
for _, tag := range input.Tags {
|
||||
if len(tag) > 64 {
|
||||
writeError(writer, 400, "invalid tag", "invalid_request")
|
||||
return
|
||||
}
|
||||
}
|
||||
if _, err := app.agent.credential(input.CredentialRef); err != nil {
|
||||
writeError(writer, http.StatusBadRequest, "credential reference is unavailable", "credential_unavailable")
|
||||
return
|
||||
}
|
||||
tags, _ := json.Marshal(input.Tags)
|
||||
server := Server{ID: newID(), Name: input.Name, Hostname: input.Hostname, AgentEndpoint: input.AgentEndpoint,
|
||||
CredentialRef: input.CredentialRef, Tags: tags, Status: "unknown", CreatedAt: time.Now()}
|
||||
if err := app.store.CreateServer(request.Context(), server); err != nil {
|
||||
writeInternal(writer, err)
|
||||
return
|
||||
}
|
||||
app.store.Audit(request.Context(), &user, server.ID, "server.create", server.Name, nil,
|
||||
map[string]any{"name": server.Name, "hostname": server.Hostname, "agent_endpoint": server.AgentEndpoint},
|
||||
"success", app.clientIP(request))
|
||||
writeJSON(writer, http.StatusCreated, server)
|
||||
}
|
||||
|
||||
func (app *App) listAudit(writer http.ResponseWriter, request *http.Request) {
|
||||
limit := queryLimit(request, 100, 500)
|
||||
events, err := app.store.AuditEvents(request.Context(), limit)
|
||||
if err != nil {
|
||||
writeInternal(writer, err)
|
||||
return
|
||||
}
|
||||
writeJSON(writer, http.StatusOK, events)
|
||||
}
|
||||
|
||||
func (app *App) serverRoute(writer http.ResponseWriter, request *http.Request, user User, csrfHash []byte) {
|
||||
remainder := strings.TrimPrefix(request.URL.Path, "/api/v1/servers/")
|
||||
parts := strings.Split(remainder, "/")
|
||||
if len(parts) < 2 || parts[0] == "" {
|
||||
writeError(writer, 404, "not found", "not_found")
|
||||
return
|
||||
}
|
||||
server, err := app.store.Server(request.Context(), parts[0])
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
writeError(writer, 404, "server not found", "not_found")
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
writeInternal(writer, err)
|
||||
return
|
||||
}
|
||||
|
||||
readRoutes := map[string]string{
|
||||
"status": "/v1/system/status", "interfaces": "/v1/network/interfaces", "routes": "/v1/network/routes",
|
||||
"listeners": "/v1/network/listeners", "connections": "/v1/network/connections",
|
||||
"firewall": "/v1/firewall/rules", "fail2ban": "/v1/fail2ban/status",
|
||||
"wireguard": "/v1/wireguard/status", "services": "/v1/services",
|
||||
}
|
||||
if len(parts) == 2 && request.Method == http.MethodGet {
|
||||
if parts[1] == "metrics" {
|
||||
samples, metricErr := app.store.Metrics(request.Context(), server.ID, queryLimit(request, 120, 1440))
|
||||
if metricErr != nil {
|
||||
writeInternal(writer, metricErr)
|
||||
return
|
||||
}
|
||||
writeJSON(writer, 200, samples)
|
||||
return
|
||||
}
|
||||
if parts[1] == "logs" {
|
||||
entries, logErr := app.store.Logs(request.Context(), server.ID, queryLimit(request, 200, 1000))
|
||||
if logErr != nil {
|
||||
writeInternal(writer, logErr)
|
||||
return
|
||||
}
|
||||
writeJSON(writer, 200, entries)
|
||||
return
|
||||
}
|
||||
if agentPath, found := readRoutes[parts[1]]; found {
|
||||
app.proxyRead(writer, request, server, agentPath)
|
||||
return
|
||||
}
|
||||
}
|
||||
if len(parts) == 3 && request.Method == http.MethodGet && parts[1] == "fail2ban" && validSimpleName(parts[2]) {
|
||||
app.proxyRead(writer, request, server, "/v1/fail2ban/jails/"+url.PathEscape(parts[2]))
|
||||
return
|
||||
}
|
||||
if len(parts) == 3 && request.Method == http.MethodPost && parts[1] == "fail2ban" && (parts[2] == "ban" || parts[2] == "unban") {
|
||||
if !app.authorize(writer, request, user, csrfHash, RoleOperator) {
|
||||
return
|
||||
}
|
||||
app.fail2banMutation(writer, request, user, server, parts[2])
|
||||
return
|
||||
}
|
||||
if len(parts) == 4 && request.Method == http.MethodPost && parts[1] == "services" && parts[3] == "restart" && validServiceName(parts[2]) {
|
||||
if !app.authorize(writer, request, user, csrfHash, RoleOperator) {
|
||||
return
|
||||
}
|
||||
app.serviceMutation(writer, request, user, server, parts[2])
|
||||
return
|
||||
}
|
||||
writeError(writer, 404, "not found", "not_found")
|
||||
}
|
||||
|
||||
func (app *App) proxyRead(writer http.ResponseWriter, request *http.Request, server Server, path string) {
|
||||
status, body, err := app.agent.Do(request.Context(), server, http.MethodGet, path, nil)
|
||||
if err != nil {
|
||||
log.Printf("agent read %s: %v", server.ID, err)
|
||||
writeError(writer, 502, "agent unavailable", "agent_unavailable")
|
||||
return
|
||||
}
|
||||
writeAgentResponse(writer, status, body)
|
||||
}
|
||||
|
||||
func (app *App) fail2banMutation(writer http.ResponseWriter, request *http.Request, user User, server Server, action string) {
|
||||
var input struct {
|
||||
Jail string `json:"jail"`
|
||||
IP string `json:"ip"`
|
||||
}
|
||||
if decodeJSON(writer, request, &input) != nil || !validSimpleName(input.Jail) || net.ParseIP(input.IP) == nil {
|
||||
writeError(writer, 400, "invalid jail or IP address", "invalid_request")
|
||||
return
|
||||
}
|
||||
body, _ := json.Marshal(input)
|
||||
status, response, err := app.agent.Do(request.Context(), server, http.MethodPost, "/v1/fail2ban/"+action, body)
|
||||
result := "success"
|
||||
if err != nil || status >= 300 {
|
||||
result = "failure"
|
||||
}
|
||||
app.store.Audit(request.Context(), &user, server.ID, "fail2ban."+action, input.Jail+":"+input.IP,
|
||||
nil, input, result, app.clientIP(request))
|
||||
if err != nil {
|
||||
writeError(writer, 502, "agent unavailable", "agent_unavailable")
|
||||
return
|
||||
}
|
||||
writeAgentResponse(writer, status, response)
|
||||
}
|
||||
|
||||
func (app *App) serviceMutation(writer http.ResponseWriter, request *http.Request, user User, server Server, name string) {
|
||||
status, response, err := app.agent.Do(request.Context(), server, http.MethodPost,
|
||||
"/v1/services/"+url.PathEscape(name)+"/restart", []byte("{}"))
|
||||
result := "success"
|
||||
if err != nil || status >= 300 {
|
||||
result = "failure"
|
||||
}
|
||||
app.store.Audit(request.Context(), &user, server.ID, "service.restart", name, nil,
|
||||
map[string]string{"requested": "restart"}, result, app.clientIP(request))
|
||||
if err != nil {
|
||||
writeError(writer, 502, "agent unavailable", "agent_unavailable")
|
||||
return
|
||||
}
|
||||
writeAgentResponse(writer, status, response)
|
||||
}
|
||||
|
||||
func (app *App) clientIP(request *http.Request) string {
|
||||
if app.config.TrustProxy {
|
||||
if value := request.Header.Get("X-Real-IP"); net.ParseIP(value) != nil {
|
||||
return value
|
||||
}
|
||||
}
|
||||
host, _, err := net.SplitHostPort(request.RemoteAddr)
|
||||
if err == nil {
|
||||
return host
|
||||
}
|
||||
return request.RemoteAddr
|
||||
}
|
||||
|
||||
func validSimpleName(value string) bool {
|
||||
if len(value) < 1 || len(value) > 64 {
|
||||
return false
|
||||
}
|
||||
for _, char := range value {
|
||||
if !(char >= 'a' && char <= 'z') && !(char >= 'A' && char <= 'Z') &&
|
||||
!(char >= '0' && char <= '9') && !strings.ContainsRune("._-", char) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func validServiceName(value string) bool {
|
||||
if len(value) < 1 || len(value) > 128 {
|
||||
return false
|
||||
}
|
||||
for _, char := range value {
|
||||
if !(char >= 'a' && char <= 'z') && !(char >= 'A' && char <= 'Z') &&
|
||||
!(char >= '0' && char <= '9') && !strings.ContainsRune("._-@:", char) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func decodeJSON(writer http.ResponseWriter, request *http.Request, target any) error {
|
||||
request.Body = http.MaxBytesReader(writer, request.Body, maxRequestBody)
|
||||
decoder := json.NewDecoder(request.Body)
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(target); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) {
|
||||
return errors.New("multiple JSON values")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func queryLimit(request *http.Request, fallback, maximum int) int {
|
||||
value, err := strconv.Atoi(request.URL.Query().Get("limit"))
|
||||
if err != nil || value < 1 {
|
||||
return fallback
|
||||
}
|
||||
if value > maximum {
|
||||
return maximum
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func writeAgentResponse(writer http.ResponseWriter, status int, payload []byte) {
|
||||
if status < 100 || status > 599 {
|
||||
status = http.StatusBadGateway
|
||||
}
|
||||
writer.Header().Set("Content-Type", "application/json")
|
||||
writer.WriteHeader(status)
|
||||
_, _ = writer.Write(payload)
|
||||
}
|
||||
|
||||
func writeJSON(writer http.ResponseWriter, status int, value any) {
|
||||
writer.Header().Set("Content-Type", "application/json")
|
||||
writer.WriteHeader(status)
|
||||
_ = json.NewEncoder(writer).Encode(value)
|
||||
}
|
||||
|
||||
func writeError(writer http.ResponseWriter, status int, message, code string) {
|
||||
writeJSON(writer, status, map[string]string{"error": message, "code": code})
|
||||
}
|
||||
|
||||
func writeInternal(writer http.ResponseWriter, err error) {
|
||||
log.Printf("internal request error: %v", err)
|
||||
writeError(writer, http.StatusInternalServerError, "internal error", "internal_error")
|
||||
}
|
||||
77
control-plane/backend/api_test.go
Normal file
77
control-plane/backend/api_test.go
Normal file
@@ -0,0 +1,77 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestStrictJSON(t *testing.T) {
|
||||
type payload struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
for _, test := range []struct {
|
||||
body string
|
||||
valid bool
|
||||
}{
|
||||
{`{"name":"vps"}`, true},
|
||||
{`{"name":"vps","secret":"leak"}`, false},
|
||||
{`{"name":"vps"}{"name":"second"}`, false},
|
||||
} {
|
||||
request := httptest.NewRequest("POST", "/", strings.NewReader(test.body))
|
||||
writer := httptest.NewRecorder()
|
||||
var value payload
|
||||
err := decodeJSON(writer, request, &value)
|
||||
if (err == nil) != test.valid {
|
||||
t.Fatalf("body %q valid=%v err=%v", test.body, test.valid, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoginLimiterIsBounded(t *testing.T) {
|
||||
limiter := NewLoginLimiter(2, time.Minute)
|
||||
now := time.Now()
|
||||
if !limiter.Allow("ip", now) || !limiter.Allow("ip", now) {
|
||||
t.Fatal("initial attempts rejected")
|
||||
}
|
||||
if limiter.Allow("ip", now) {
|
||||
t.Fatal("limit not enforced")
|
||||
}
|
||||
if !limiter.Allow("ip", now.Add(time.Minute)) {
|
||||
t.Fatal("window did not reset")
|
||||
}
|
||||
limiter.maxKeys = 1
|
||||
if limiter.Allow("second-ip", now) {
|
||||
t.Fatal("bounded limiter accepted an untracked key")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidators(t *testing.T) {
|
||||
if !validSimpleName("nginx-http-auth") || validSimpleName("jail;shutdown") {
|
||||
t.Fatal("jail validation failed")
|
||||
}
|
||||
if !validServiceName("wg-quick@wg0") || validServiceName("nginx/restart") {
|
||||
t.Fatal("service validation failed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPercent(t *testing.T) {
|
||||
if percent(25, 100) != 25 || percent(1, 0) != 0 {
|
||||
t.Fatal("invalid percentage")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedactLogMessage(t *testing.T) {
|
||||
input := `Authorization: Bearer abc123 password="hunter2" token=xyz&safe=1`
|
||||
want := `Authorization: Bearer [REDACTED] password=[REDACTED] token=[REDACTED]&safe=1`
|
||||
if got := redactLogMessage(input); got != want {
|
||||
t.Fatalf("redaction mismatch: got %q want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTruncateUTF8(t *testing.T) {
|
||||
if got := truncateUTF8("привет", 5); got != "пр" {
|
||||
t.Fatalf("unexpected UTF-8 truncation %q", got)
|
||||
}
|
||||
}
|
||||
113
control-plane/backend/auth.go
Normal file
113
control-plane/backend/auth.go
Normal file
@@ -0,0 +1,113 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"crypto/subtle"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/crypto/argon2"
|
||||
)
|
||||
|
||||
const (
|
||||
RoleViewer = "viewer"
|
||||
RoleOperator = "operator"
|
||||
RoleAdministrator = "administrator"
|
||||
)
|
||||
|
||||
type User struct {
|
||||
ID string `json:"id"`
|
||||
Username string `json:"username"`
|
||||
Role string `json:"role"`
|
||||
}
|
||||
|
||||
func HashPassword(password string) (string, error) {
|
||||
if len(password) < 12 || len(password) > 1024 {
|
||||
return "", errors.New("password must be between 12 and 1024 bytes")
|
||||
}
|
||||
salt := make([]byte, 16)
|
||||
if _, err := rand.Read(salt); err != nil {
|
||||
return "", err
|
||||
}
|
||||
const memory = 64 * 1024
|
||||
const iterations = 3
|
||||
const parallelism = 2
|
||||
hash := argon2.IDKey([]byte(password), salt, iterations, memory, parallelism, 32)
|
||||
return fmt.Sprintf("$argon2id$v=19$m=%d,t=%d,p=%d$%s$%s", memory, iterations, parallelism,
|
||||
base64.RawStdEncoding.EncodeToString(salt), base64.RawStdEncoding.EncodeToString(hash)), nil
|
||||
}
|
||||
|
||||
func VerifyPassword(encoded, password string) bool {
|
||||
parts := strings.Split(encoded, "$")
|
||||
if len(parts) != 6 || parts[1] != "argon2id" || parts[2] != "v=19" {
|
||||
return false
|
||||
}
|
||||
var memory uint32
|
||||
var iterations uint32
|
||||
var parallelism uint8
|
||||
if _, err := fmt.Sscanf(parts[3], "m=%d,t=%d,p=%d", &memory, &iterations, ¶llelism); err != nil {
|
||||
return false
|
||||
}
|
||||
if memory > 128*1024 || iterations > 10 || parallelism > 8 {
|
||||
return false
|
||||
}
|
||||
salt, err := base64.RawStdEncoding.DecodeString(parts[4])
|
||||
if err != nil || len(salt) < 16 {
|
||||
return false
|
||||
}
|
||||
expected, err := base64.RawStdEncoding.DecodeString(parts[5])
|
||||
if err != nil || len(expected) != 32 {
|
||||
return false
|
||||
}
|
||||
actual := argon2.IDKey([]byte(password), salt, iterations, memory, parallelism, uint32(len(expected)))
|
||||
return subtle.ConstantTimeCompare(actual, expected) == 1
|
||||
}
|
||||
|
||||
func randomToken() (string, []byte, error) {
|
||||
raw := make([]byte, 32)
|
||||
if _, err := rand.Read(raw); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
token := base64.RawURLEncoding.EncodeToString(raw)
|
||||
digest := sha256.Sum256([]byte(token))
|
||||
return token, digest[:], nil
|
||||
}
|
||||
|
||||
func tokenHash(token string) []byte {
|
||||
digest := sha256.Sum256([]byte(token))
|
||||
return digest[:]
|
||||
}
|
||||
|
||||
func validRole(role string) bool {
|
||||
return role == RoleViewer || role == RoleOperator || role == RoleAdministrator
|
||||
}
|
||||
|
||||
func permits(role, required string) bool {
|
||||
rank := map[string]int{RoleViewer: 1, RoleOperator: 2, RoleAdministrator: 3}
|
||||
return rank[role] >= rank[required] && rank[required] > 0
|
||||
}
|
||||
|
||||
func validUsername(username string) bool {
|
||||
if len(username) < 3 || len(username) > 64 {
|
||||
return false
|
||||
}
|
||||
for _, char := range username {
|
||||
if !(char >= 'a' && char <= 'z') && !(char >= 'A' && char <= 'Z') &&
|
||||
!(char >= '0' && char <= '9') && !strings.ContainsRune("._-", char) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func parseInt(value string, fallback int) int {
|
||||
parsed, err := strconv.Atoi(value)
|
||||
if err != nil {
|
||||
return fallback
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
52
control-plane/backend/auth_test.go
Normal file
52
control-plane/backend/auth_test.go
Normal file
@@ -0,0 +1,52 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPasswordHashAndVerify(t *testing.T) {
|
||||
hash, err := HashPassword("correct horse battery staple")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if hash == "correct horse battery staple" {
|
||||
t.Fatal("password stored in plaintext")
|
||||
}
|
||||
if !VerifyPassword(hash, "correct horse battery staple") {
|
||||
t.Fatal("correct password rejected")
|
||||
}
|
||||
if VerifyPassword(hash, "wrong password") {
|
||||
t.Fatal("wrong password accepted")
|
||||
}
|
||||
if VerifyPassword("malformed", "correct horse battery staple") {
|
||||
t.Fatal("malformed hash accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRBAC(t *testing.T) {
|
||||
if !permits(RoleAdministrator, RoleOperator) {
|
||||
t.Fatal("administrator must inherit operator")
|
||||
}
|
||||
if permits(RoleViewer, RoleOperator) {
|
||||
t.Fatal("viewer must not mutate")
|
||||
}
|
||||
if !permits(RoleOperator, RoleViewer) {
|
||||
t.Fatal("operator must read")
|
||||
}
|
||||
if permits("unknown", RoleViewer) {
|
||||
t.Fatal("unknown role allowed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUsernameValidation(t *testing.T) {
|
||||
for _, value := range []string{"admin", "ops.user", "alice-2"} {
|
||||
if !validUsername(value) {
|
||||
t.Fatalf("valid username rejected: %s", value)
|
||||
}
|
||||
}
|
||||
for _, value := range []string{"x", "admin@example", "../../root"} {
|
||||
if validUsername(value) {
|
||||
t.Fatalf("invalid username accepted: %s", value)
|
||||
}
|
||||
}
|
||||
}
|
||||
126
control-plane/backend/config.go
Normal file
126
control-plane/backend/config.go
Normal file
@@ -0,0 +1,126 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
ListenAddress string
|
||||
DatabaseURL string
|
||||
CookieSecure bool
|
||||
TrustProxy bool
|
||||
SessionTTL time.Duration
|
||||
CredentialsDir string
|
||||
AdminUsername string
|
||||
AdminPasswordFile string
|
||||
BootstrapServerName string
|
||||
BootstrapEndpoint string
|
||||
BootstrapCredential string
|
||||
MetricsRetention time.Duration
|
||||
LogsRetention time.Duration
|
||||
IntegrationTokenFile string
|
||||
}
|
||||
|
||||
func LoadConfig() (Config, error) {
|
||||
config := Config{
|
||||
ListenAddress: envOr("LISTEN_ADDRESS", ":8080"),
|
||||
DatabaseURL: os.Getenv("DATABASE_URL"),
|
||||
CookieSecure: envBool("COOKIE_SECURE", true),
|
||||
TrustProxy: envBool("TRUST_PROXY", false),
|
||||
SessionTTL: envDuration("SESSION_TTL", 12*time.Hour),
|
||||
CredentialsDir: envOr("AGENT_CREDENTIALS_DIR", "/run/secrets"),
|
||||
AdminUsername: envOr("INITIAL_ADMIN_USERNAME", "admin"),
|
||||
AdminPasswordFile: os.Getenv("INITIAL_ADMIN_PASSWORD_FILE"),
|
||||
BootstrapServerName: os.Getenv("BOOTSTRAP_SERVER_NAME"),
|
||||
BootstrapEndpoint: os.Getenv("BOOTSTRAP_SERVER_ENDPOINT"),
|
||||
BootstrapCredential: os.Getenv("BOOTSTRAP_AGENT_CREDENTIAL_REF"),
|
||||
MetricsRetention: envDuration("METRICS_RETENTION", 30*24*time.Hour),
|
||||
LogsRetention: envDuration("LOGS_RETENTION", 7*24*time.Hour),
|
||||
IntegrationTokenFile: os.Getenv("INTEGRATION_TOKEN_FILE"),
|
||||
}
|
||||
if config.DatabaseURL == "" {
|
||||
config.DatabaseURL = buildDatabaseURL()
|
||||
}
|
||||
if config.DatabaseURL == "" {
|
||||
return Config{}, errors.New("DATABASE_URL or DB_HOST/DB_PASSWORD_FILE is required")
|
||||
}
|
||||
if config.SessionTTL < 5*time.Minute || config.SessionTTL > 7*24*time.Hour {
|
||||
return Config{}, errors.New("SESSION_TTL must be between 5m and 168h")
|
||||
}
|
||||
if config.MetricsRetention < 24*time.Hour {
|
||||
return Config{}, errors.New("METRICS_RETENTION must be at least 24h")
|
||||
}
|
||||
if config.LogsRetention < time.Hour || config.LogsRetention > 90*24*time.Hour {
|
||||
return Config{}, errors.New("LOGS_RETENTION must be between 1h and 2160h")
|
||||
}
|
||||
if config.BootstrapEndpoint != "" {
|
||||
if err := validateAgentEndpoint(config.BootstrapEndpoint); err != nil {
|
||||
return Config{}, fmt.Errorf("bootstrap endpoint: %w", err)
|
||||
}
|
||||
if !validCredentialRef(config.BootstrapCredential) {
|
||||
return Config{}, errors.New("invalid BOOTSTRAP_AGENT_CREDENTIAL_REF")
|
||||
}
|
||||
}
|
||||
return config, nil
|
||||
}
|
||||
|
||||
func buildDatabaseURL() string {
|
||||
host := os.Getenv("DB_HOST")
|
||||
passwordFile := os.Getenv("DB_PASSWORD_FILE")
|
||||
if host == "" || passwordFile == "" {
|
||||
return ""
|
||||
}
|
||||
password, err := os.ReadFile(filepath.Clean(passwordFile))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
user := envOr("DB_USER", "vps_control")
|
||||
database := envOr("DB_NAME", "vps_control")
|
||||
values := url.Values{"sslmode": {envOr("DB_SSLMODE", "disable")}}
|
||||
return (&url.URL{
|
||||
Scheme: "postgres",
|
||||
User: url.UserPassword(user, strings.TrimSpace(string(password))),
|
||||
Host: net.JoinHostPort(host, envOr("DB_PORT", "5432")),
|
||||
Path: database,
|
||||
RawQuery: values.Encode(),
|
||||
}).String()
|
||||
}
|
||||
|
||||
func envOr(name, fallback string) string {
|
||||
if value := os.Getenv(name); value != "" {
|
||||
return value
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func envBool(name string, fallback bool) bool {
|
||||
value := os.Getenv(name)
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
parsed, err := strconv.ParseBool(value)
|
||||
if err != nil {
|
||||
return fallback
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
func envDuration(name string, fallback time.Duration) time.Duration {
|
||||
value := os.Getenv(name)
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
parsed, err := time.ParseDuration(value)
|
||||
if err != nil {
|
||||
return fallback
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
572
control-plane/backend/db.go
Normal file
572
control-plane/backend/db.go
Normal file
@@ -0,0 +1,572 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
const schema = `
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id text PRIMARY KEY,
|
||||
username text UNIQUE NOT NULL,
|
||||
password_hash text NOT NULL,
|
||||
role text NOT NULL CHECK (role IN ('viewer','operator','administrator')),
|
||||
disabled boolean NOT NULL DEFAULT false,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
id_hash bytea PRIMARY KEY,
|
||||
user_id text NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
csrf_hash bytea NOT NULL,
|
||||
source_ip text NOT NULL,
|
||||
expires_at timestamptz NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS sessions_expires_idx ON sessions(expires_at);
|
||||
CREATE TABLE IF NOT EXISTS servers (
|
||||
id text PRIMARY KEY,
|
||||
name text NOT NULL,
|
||||
hostname text NOT NULL DEFAULT '',
|
||||
agent_endpoint text NOT NULL,
|
||||
credential_ref text NOT NULL,
|
||||
tags jsonb NOT NULL DEFAULT '[]'::jsonb,
|
||||
status text NOT NULL DEFAULT 'unknown',
|
||||
last_seen timestamptz,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS metric_samples (
|
||||
server_id text NOT NULL REFERENCES servers(id) ON DELETE CASCADE,
|
||||
sampled_at timestamptz NOT NULL DEFAULT now(),
|
||||
cpu double precision NOT NULL,
|
||||
memory double precision NOT NULL,
|
||||
disk double precision NOT NULL,
|
||||
rx bigint NOT NULL,
|
||||
tx bigint NOT NULL,
|
||||
connections integer NOT NULL,
|
||||
bans integer NOT NULL,
|
||||
PRIMARY KEY(server_id, sampled_at)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS metric_samples_time_idx ON metric_samples(sampled_at);
|
||||
CREATE TABLE IF NOT EXISTS agent_log_entries (
|
||||
server_id text NOT NULL REFERENCES servers(id) ON DELETE CASCADE,
|
||||
journal_cursor text NOT NULL,
|
||||
event_at timestamptz NOT NULL,
|
||||
unit text NOT NULL,
|
||||
priority smallint NOT NULL CHECK (priority BETWEEN 0 AND 7),
|
||||
message text NOT NULL,
|
||||
boot_id text,
|
||||
pid integer,
|
||||
ingested_at timestamptz NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY(server_id, journal_cursor)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS agent_log_entries_time_idx ON agent_log_entries(event_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS agent_log_entries_server_time_idx ON agent_log_entries(server_id, event_at DESC);
|
||||
CREATE TABLE IF NOT EXISTS audit_events (
|
||||
id bigserial PRIMARY KEY,
|
||||
user_id text REFERENCES users(id) ON DELETE SET NULL,
|
||||
username text NOT NULL,
|
||||
server_id text REFERENCES servers(id) ON DELETE SET NULL,
|
||||
action text NOT NULL,
|
||||
target text NOT NULL,
|
||||
before_state jsonb,
|
||||
after_state jsonb,
|
||||
result text NOT NULL,
|
||||
source_ip text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS audit_events_time_idx ON audit_events(created_at DESC);
|
||||
`
|
||||
|
||||
type Store struct{ pool *pgxpool.Pool }
|
||||
|
||||
type Server struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Hostname string `json:"hostname"`
|
||||
AgentEndpoint string `json:"agent_endpoint"`
|
||||
CredentialRef string `json:"-"`
|
||||
Tags json.RawMessage `json:"tags"`
|
||||
Status string `json:"status"`
|
||||
LastSeen *time.Time `json:"last_seen"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
type AuditEvent struct {
|
||||
ID int64 `json:"id"`
|
||||
Username string `json:"username"`
|
||||
ServerID *string `json:"server_id"`
|
||||
Action string `json:"action"`
|
||||
Target string `json:"target"`
|
||||
Before json.RawMessage `json:"before"`
|
||||
After json.RawMessage `json:"after"`
|
||||
Result string `json:"result"`
|
||||
SourceIP string `json:"source_ip"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
type MetricSample struct {
|
||||
SampledAt time.Time `json:"sampled_at"`
|
||||
CPU float64 `json:"cpu"`
|
||||
Memory float64 `json:"memory"`
|
||||
Disk float64 `json:"disk"`
|
||||
RX int64 `json:"rx"`
|
||||
TX int64 `json:"tx"`
|
||||
Connections int `json:"connections"`
|
||||
Bans int `json:"bans"`
|
||||
}
|
||||
|
||||
type AgentLogEntry struct {
|
||||
ServerID string `json:"server_id"`
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
Unit string `json:"unit"`
|
||||
Priority int `json:"priority"`
|
||||
Message string `json:"message"`
|
||||
BootID *string `json:"boot_id,omitempty"`
|
||||
PID *int `json:"pid,omitempty"`
|
||||
}
|
||||
|
||||
type MetricPoint struct {
|
||||
ServerID string
|
||||
ServerName string
|
||||
Timestamp time.Time
|
||||
Value float64
|
||||
}
|
||||
|
||||
type IntegrationLog struct {
|
||||
ServerID string
|
||||
ServerName string
|
||||
Timestamp time.Time
|
||||
Unit string
|
||||
Priority int
|
||||
Message string
|
||||
}
|
||||
|
||||
func OpenStore(ctx context.Context, databaseURL string) (*Store, error) {
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < 30; attempt++ {
|
||||
pool, err := pgxpool.New(ctx, databaseURL)
|
||||
if err == nil {
|
||||
err = pool.Ping(ctx)
|
||||
}
|
||||
if err == nil {
|
||||
store := &Store{pool: pool}
|
||||
if _, err = pool.Exec(ctx, schema); err == nil {
|
||||
return store, nil
|
||||
}
|
||||
pool.Close()
|
||||
}
|
||||
lastErr = err
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-time.After(time.Second):
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("connect/migrate database: %w", lastErr)
|
||||
}
|
||||
|
||||
func (store *Store) Close() { store.pool.Close() }
|
||||
|
||||
func (store *Store) Bootstrap(ctx context.Context, config Config) error {
|
||||
var count int
|
||||
if err := store.pool.QueryRow(ctx, "SELECT count(*) FROM users").Scan(&count); err != nil {
|
||||
return err
|
||||
}
|
||||
if count == 0 {
|
||||
if !validUsername(config.AdminUsername) || config.AdminPasswordFile == "" {
|
||||
return errors.New("first start requires INITIAL_ADMIN_USERNAME and INITIAL_ADMIN_PASSWORD_FILE")
|
||||
}
|
||||
password, err := os.ReadFile(config.AdminPasswordFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read initial admin password: %w", err)
|
||||
}
|
||||
hash, err := HashPassword(strings.TrimSpace(string(password)))
|
||||
if err != nil {
|
||||
return fmt.Errorf("initial admin password: %w", err)
|
||||
}
|
||||
if _, err = store.pool.Exec(ctx,
|
||||
"INSERT INTO users(id, username, password_hash, role) VALUES($1,$2,$3,$4)",
|
||||
newID(), config.AdminUsername, hash, RoleAdministrator); err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("created initial administrator %q", config.AdminUsername)
|
||||
}
|
||||
if config.BootstrapEndpoint != "" {
|
||||
_, err := store.pool.Exec(ctx, `INSERT INTO servers(id,name,agent_endpoint,credential_ref)
|
||||
SELECT $1,$2,$3,$4 WHERE NOT EXISTS (SELECT 1 FROM servers)`,
|
||||
newID(), config.BootstrapServerName, config.BootstrapEndpoint, config.BootstrapCredential)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *Store) LoginUser(ctx context.Context, username string) (User, string, bool, error) {
|
||||
var user User
|
||||
var hash string
|
||||
var disabled bool
|
||||
err := store.pool.QueryRow(ctx,
|
||||
"SELECT id,username,role,password_hash,disabled FROM users WHERE username=$1", username).
|
||||
Scan(&user.ID, &user.Username, &user.Role, &hash, &disabled)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return User{}, "", false, nil
|
||||
}
|
||||
return user, hash, disabled, err
|
||||
}
|
||||
|
||||
func (store *Store) CreateSession(ctx context.Context, userID, sourceIP string, ttl time.Duration) (string, string, error) {
|
||||
session, sessionHash, err := randomToken()
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
csrf, csrfHash, err := randomToken()
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
_, err = store.pool.Exec(ctx,
|
||||
"INSERT INTO sessions(id_hash,user_id,csrf_hash,source_ip,expires_at) VALUES($1,$2,$3,$4,$5)",
|
||||
sessionHash, userID, csrfHash, sourceIP, time.Now().Add(ttl))
|
||||
return session, csrf, err
|
||||
}
|
||||
|
||||
func (store *Store) SessionUser(ctx context.Context, token string) (User, []byte, error) {
|
||||
var user User
|
||||
var csrfHash []byte
|
||||
err := store.pool.QueryRow(ctx, `SELECT u.id,u.username,u.role,s.csrf_hash
|
||||
FROM sessions s JOIN users u ON u.id=s.user_id
|
||||
WHERE s.id_hash=$1 AND s.expires_at>now() AND NOT u.disabled`, tokenHash(token)).
|
||||
Scan(&user.ID, &user.Username, &user.Role, &csrfHash)
|
||||
return user, csrfHash, err
|
||||
}
|
||||
|
||||
func (store *Store) DeleteSession(ctx context.Context, token string) error {
|
||||
_, err := store.pool.Exec(ctx, "DELETE FROM sessions WHERE id_hash=$1", tokenHash(token))
|
||||
return err
|
||||
}
|
||||
|
||||
func (store *Store) Servers(ctx context.Context) ([]Server, error) {
|
||||
rows, err := store.pool.Query(ctx, `SELECT id,name,hostname,agent_endpoint,credential_ref,tags,status,last_seen,created_at
|
||||
FROM servers ORDER BY name`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
servers := []Server{}
|
||||
for rows.Next() {
|
||||
var server Server
|
||||
if err = rows.Scan(&server.ID, &server.Name, &server.Hostname, &server.AgentEndpoint,
|
||||
&server.CredentialRef, &server.Tags, &server.Status, &server.LastSeen, &server.CreatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
servers = append(servers, server)
|
||||
}
|
||||
return servers, rows.Err()
|
||||
}
|
||||
|
||||
func (store *Store) Server(ctx context.Context, id string) (Server, error) {
|
||||
var server Server
|
||||
err := store.pool.QueryRow(ctx, `SELECT id,name,hostname,agent_endpoint,credential_ref,tags,status,last_seen,created_at
|
||||
FROM servers WHERE id=$1`, id).Scan(&server.ID, &server.Name, &server.Hostname, &server.AgentEndpoint,
|
||||
&server.CredentialRef, &server.Tags, &server.Status, &server.LastSeen, &server.CreatedAt)
|
||||
return server, err
|
||||
}
|
||||
|
||||
func (store *Store) CreateServer(ctx context.Context, server Server) error {
|
||||
_, err := store.pool.Exec(ctx, `INSERT INTO servers(id,name,hostname,agent_endpoint,credential_ref,tags)
|
||||
VALUES($1,$2,$3,$4,$5,$6)`, server.ID, server.Name, server.Hostname,
|
||||
server.AgentEndpoint, server.CredentialRef, server.Tags)
|
||||
return err
|
||||
}
|
||||
|
||||
func (store *Store) Audit(ctx context.Context, user *User, serverID, action, target string,
|
||||
before, after any, result, sourceIP string) {
|
||||
var userID any
|
||||
username := "anonymous"
|
||||
if user != nil {
|
||||
userID, username = user.ID, user.Username
|
||||
}
|
||||
beforeJSON, _ := json.Marshal(before)
|
||||
afterJSON, _ := json.Marshal(after)
|
||||
if before == nil {
|
||||
beforeJSON = nil
|
||||
}
|
||||
if after == nil {
|
||||
afterJSON = nil
|
||||
}
|
||||
if _, err := store.pool.Exec(ctx, `INSERT INTO audit_events
|
||||
(user_id,username,server_id,action,target,before_state,after_state,result,source_ip)
|
||||
VALUES($1,$2,NULLIF($3,''),$4,$5,$6,$7,$8,$9)`, userID, username, serverID,
|
||||
action, target, beforeJSON, afterJSON, result, sourceIP); err != nil {
|
||||
log.Printf("write audit event: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (store *Store) AuditEvents(ctx context.Context, limit int) ([]AuditEvent, error) {
|
||||
rows, err := store.pool.Query(ctx, `SELECT id,username,server_id,action,target,
|
||||
COALESCE(before_state,'null'::jsonb),COALESCE(after_state,'null'::jsonb),result,source_ip,created_at
|
||||
FROM audit_events ORDER BY created_at DESC LIMIT $1`, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
events := []AuditEvent{}
|
||||
for rows.Next() {
|
||||
var event AuditEvent
|
||||
if err = rows.Scan(&event.ID, &event.Username, &event.ServerID, &event.Action, &event.Target,
|
||||
&event.Before, &event.After, &event.Result, &event.SourceIP, &event.CreatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
events = append(events, event)
|
||||
}
|
||||
return events, rows.Err()
|
||||
}
|
||||
|
||||
func (store *Store) Metrics(ctx context.Context, serverID string, limit int) ([]MetricSample, error) {
|
||||
rows, err := store.pool.Query(ctx, `SELECT sampled_at,cpu,memory,disk,rx,tx,connections,bans
|
||||
FROM metric_samples WHERE server_id=$1 ORDER BY sampled_at DESC LIMIT $2`, serverID, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
samples := []MetricSample{}
|
||||
for rows.Next() {
|
||||
var sample MetricSample
|
||||
if err = rows.Scan(&sample.SampledAt, &sample.CPU, &sample.Memory, &sample.Disk,
|
||||
&sample.RX, &sample.TX, &sample.Connections, &sample.Bans); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
samples = append(samples, sample)
|
||||
}
|
||||
return samples, rows.Err()
|
||||
}
|
||||
|
||||
func (store *Store) Logs(ctx context.Context, serverID string, limit int) ([]AgentLogEntry, error) {
|
||||
rows, err := store.pool.Query(ctx, `SELECT server_id,event_at,unit,priority,message,boot_id,pid
|
||||
FROM agent_log_entries WHERE server_id=$1 ORDER BY event_at DESC LIMIT $2`, serverID, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
entries := []AgentLogEntry{}
|
||||
for rows.Next() {
|
||||
var entry AgentLogEntry
|
||||
if err = rows.Scan(&entry.ServerID, &entry.Timestamp, &entry.Unit, &entry.Priority,
|
||||
&entry.Message, &entry.BootID, &entry.PID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
return entries, rows.Err()
|
||||
}
|
||||
|
||||
func (store *Store) MetricSeries(ctx context.Context) ([]map[string]string, error) {
|
||||
rows, err := store.pool.Query(ctx, "SELECT id,name FROM servers ORDER BY id LIMIT 1000")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
servers := [][2]string{}
|
||||
for rows.Next() {
|
||||
var server [2]string
|
||||
if err = rows.Scan(&server[0], &server[1]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
servers = append(servers, server)
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
names := make([]string, 0, len(metricColumns))
|
||||
for name := range metricColumns {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
series := make([]map[string]string, 0, len(servers)*len(names))
|
||||
for _, server := range servers {
|
||||
for _, name := range names {
|
||||
series = append(series, map[string]string{
|
||||
"__name__": name, "server_id": server[0], "server_name": server[1],
|
||||
})
|
||||
}
|
||||
}
|
||||
return series, nil
|
||||
}
|
||||
|
||||
func (store *Store) MetricLabelValues(ctx context.Context, label string) ([]string, error) {
|
||||
column := ""
|
||||
switch label {
|
||||
case "server_id":
|
||||
column = "id"
|
||||
case "server_name":
|
||||
column = "name"
|
||||
default:
|
||||
return nil, errors.New("unsupported metric label")
|
||||
}
|
||||
rows, err := store.pool.Query(ctx, "SELECT "+column+" FROM servers ORDER BY "+column+" LIMIT 10000")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
values := []string{}
|
||||
for rows.Next() {
|
||||
var value string
|
||||
if err = rows.Scan(&value); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
values = append(values, value)
|
||||
}
|
||||
return values, rows.Err()
|
||||
}
|
||||
|
||||
func (store *Store) InstantMetric(ctx context.Context, query metricQuery, at time.Time) ([]MetricPoint, error) {
|
||||
column, found := metricColumns[query.Name]
|
||||
if !found {
|
||||
return nil, errors.New("unsupported metric")
|
||||
}
|
||||
statement := fmt.Sprintf(`SELECT DISTINCT ON (m.server_id)
|
||||
m.server_id,s.name,m.sampled_at,m.%s::double precision
|
||||
FROM metric_samples m JOIN servers s ON s.id=m.server_id
|
||||
WHERE m.sampled_at <= $1 AND ($2='' OR s.id=$2) AND ($3='' OR s.name=$3)
|
||||
ORDER BY m.server_id,m.sampled_at DESC LIMIT 10000`, column)
|
||||
rows, err := store.pool.Query(ctx, statement, at, query.ServerID, query.ServerName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
return scanMetricPoints(rows)
|
||||
}
|
||||
|
||||
func (store *Store) RangeMetric(ctx context.Context, query metricQuery, start, end time.Time, step int) ([]MetricPoint, error) {
|
||||
column, found := metricColumns[query.Name]
|
||||
if !found {
|
||||
return nil, errors.New("unsupported metric")
|
||||
}
|
||||
statement := fmt.Sprintf(`SELECT m.server_id,s.name,
|
||||
to_timestamp(floor(extract(epoch FROM m.sampled_at)/$3)*$3) AS bucket,
|
||||
avg(m.%s::double precision)
|
||||
FROM metric_samples m JOIN servers s ON s.id=m.server_id
|
||||
WHERE m.sampled_at BETWEEN $1 AND $2 AND ($4='' OR s.id=$4) AND ($5='' OR s.name=$5)
|
||||
GROUP BY m.server_id,s.name,bucket ORDER BY m.server_id,bucket LIMIT 100000`, column)
|
||||
rows, err := store.pool.Query(ctx, statement, start, end, step, query.ServerID, query.ServerName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
return scanMetricPoints(rows)
|
||||
}
|
||||
|
||||
func scanMetricPoints(rows pgx.Rows) ([]MetricPoint, error) {
|
||||
points := []MetricPoint{}
|
||||
for rows.Next() {
|
||||
var point MetricPoint
|
||||
if err := rows.Scan(&point.ServerID, &point.ServerName, &point.Timestamp, &point.Value); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
points = append(points, point)
|
||||
}
|
||||
return points, rows.Err()
|
||||
}
|
||||
|
||||
func (store *Store) LogLabelValues(ctx context.Context, label string) ([]string, error) {
|
||||
column := ""
|
||||
switch label {
|
||||
case "server_id":
|
||||
column = "s.id"
|
||||
case "server_name":
|
||||
column = "s.name"
|
||||
case "unit":
|
||||
column = "l.unit"
|
||||
case "priority":
|
||||
column = "l.priority::text"
|
||||
default:
|
||||
return nil, errors.New("unsupported log label")
|
||||
}
|
||||
statement := `SELECT DISTINCT ` + column + ` AS value FROM agent_log_entries l
|
||||
JOIN servers s ON s.id=l.server_id ORDER BY value LIMIT 10000`
|
||||
rows, err := store.pool.Query(ctx, statement)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
values := []string{}
|
||||
for rows.Next() {
|
||||
var value string
|
||||
if err = rows.Scan(&value); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
values = append(values, value)
|
||||
}
|
||||
return values, rows.Err()
|
||||
}
|
||||
|
||||
func (store *Store) LogSeries(ctx context.Context) ([]map[string]string, error) {
|
||||
rows, err := store.pool.Query(ctx, `SELECT DISTINCT s.id,s.name,l.unit,l.priority
|
||||
FROM agent_log_entries l JOIN servers s ON s.id=l.server_id
|
||||
ORDER BY s.id,l.unit,l.priority LIMIT 10000`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
series := []map[string]string{}
|
||||
for rows.Next() {
|
||||
var serverID, serverName, unit string
|
||||
var priority int
|
||||
if err = rows.Scan(&serverID, &serverName, &unit, &priority); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
series = append(series, map[string]string{
|
||||
"server_id": serverID, "server_name": serverName, "unit": unit,
|
||||
"priority": strconv.Itoa(priority),
|
||||
})
|
||||
}
|
||||
return series, rows.Err()
|
||||
}
|
||||
|
||||
func (store *Store) IntegrationLogs(ctx context.Context, start, end time.Time, limit int, direction string) ([]IntegrationLog, error) {
|
||||
order := "DESC"
|
||||
if direction == "forward" {
|
||||
order = "ASC"
|
||||
}
|
||||
statement := `SELECT l.server_id,s.name,l.event_at,l.unit,l.priority,l.message
|
||||
FROM agent_log_entries l JOIN servers s ON s.id=l.server_id
|
||||
WHERE l.event_at BETWEEN $1 AND $2 ORDER BY l.event_at ` + order + ` LIMIT $3`
|
||||
rows, err := store.pool.Query(ctx, statement, start, end, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
entries := []IntegrationLog{}
|
||||
for rows.Next() {
|
||||
var entry IntegrationLog
|
||||
if err = rows.Scan(&entry.ServerID, &entry.ServerName, &entry.Timestamp, &entry.Unit,
|
||||
&entry.Priority, &entry.Message); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
return entries, rows.Err()
|
||||
}
|
||||
|
||||
func newID() string {
|
||||
var value [16]byte
|
||||
if _, err := rand.Read(value[:]); err != nil {
|
||||
panic("system random source unavailable")
|
||||
}
|
||||
value[6] = (value[6] & 0x0f) | 0x40
|
||||
value[8] = (value[8] & 0x3f) | 0x80
|
||||
return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x",
|
||||
value[0:4], value[4:6], value[6:8], value[8:10], value[10:16])
|
||||
}
|
||||
17
control-plane/backend/go.mod
Normal file
17
control-plane/backend/go.mod
Normal file
@@ -0,0 +1,17 @@
|
||||
module github.com/vps-control/vps-control/control-plane/backend
|
||||
|
||||
go 1.20
|
||||
|
||||
require (
|
||||
github.com/jackc/pgx/v5 v5.5.5
|
||||
golang.org/x/crypto v0.23.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.1 // indirect
|
||||
golang.org/x/sync v0.1.0 // indirect
|
||||
golang.org/x/sys v0.20.0 // indirect
|
||||
golang.org/x/text v0.15.0 // indirect
|
||||
)
|
||||
27
control-plane/backend/go.sum
Normal file
27
control-plane/backend/go.sum
Normal file
@@ -0,0 +1,27 @@
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
|
||||
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
|
||||
github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw=
|
||||
github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A=
|
||||
github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
|
||||
github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
|
||||
golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
|
||||
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||
golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
|
||||
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
|
||||
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
53
control-plane/backend/limiter.go
Normal file
53
control-plane/backend/limiter.go
Normal file
@@ -0,0 +1,53 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type loginEntry struct {
|
||||
window time.Time
|
||||
count int
|
||||
seen time.Time
|
||||
}
|
||||
|
||||
type LoginLimiter struct {
|
||||
mu sync.Mutex
|
||||
entries map[string]loginEntry
|
||||
limit int
|
||||
window time.Duration
|
||||
maxKeys int
|
||||
}
|
||||
|
||||
func NewLoginLimiter(limit int, window time.Duration) *LoginLimiter {
|
||||
return &LoginLimiter{entries: make(map[string]loginEntry), limit: limit, window: window, maxKeys: 10000}
|
||||
}
|
||||
|
||||
func (limiter *LoginLimiter) Allow(key string, now time.Time) bool {
|
||||
limiter.mu.Lock()
|
||||
defer limiter.mu.Unlock()
|
||||
_, known := limiter.entries[key]
|
||||
if !known && len(limiter.entries) >= limiter.maxKeys {
|
||||
cutoff := now.Add(-2 * limiter.window)
|
||||
for item, entry := range limiter.entries {
|
||||
if entry.seen.Before(cutoff) {
|
||||
delete(limiter.entries, item)
|
||||
}
|
||||
}
|
||||
if len(limiter.entries) >= limiter.maxKeys {
|
||||
return false
|
||||
}
|
||||
}
|
||||
entry := limiter.entries[key]
|
||||
if entry.window.IsZero() || now.Sub(entry.window) >= limiter.window {
|
||||
entry.window, entry.count = now, 0
|
||||
}
|
||||
entry.seen = now
|
||||
if entry.count >= limiter.limit {
|
||||
limiter.entries[key] = entry
|
||||
return false
|
||||
}
|
||||
entry.count++
|
||||
limiter.entries[key] = entry
|
||||
return true
|
||||
}
|
||||
52
control-plane/backend/main.go
Normal file
52
control-plane/backend/main.go
Normal file
@@ -0,0 +1,52 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
func main() {
|
||||
config, err := LoadConfig()
|
||||
if err != nil {
|
||||
log.Fatalf("configuration: %v", err)
|
||||
}
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
store, err := OpenStore(ctx, config.DatabaseURL)
|
||||
if err != nil {
|
||||
log.Fatalf("database: %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
if err = store.Bootstrap(ctx, config); err != nil {
|
||||
log.Fatalf("bootstrap: %v", err)
|
||||
}
|
||||
agent := NewAgentClient(config.CredentialsDir)
|
||||
app, err := NewApp(store, agent, config)
|
||||
if err != nil {
|
||||
log.Fatalf("application: %v", err)
|
||||
}
|
||||
server := &http.Server{
|
||||
Addr: config.ListenAddress, Handler: app,
|
||||
ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 10 * time.Second,
|
||||
WriteTimeout: 15 * time.Second, IdleTimeout: 60 * time.Second,
|
||||
MaxHeaderBytes: 16 << 10,
|
||||
}
|
||||
poller := &Poller{store: store, agent: agent, metricRetention: config.MetricsRetention, logRetention: config.LogsRetention}
|
||||
go poller.Run(ctx)
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = server.Shutdown(shutdownCtx)
|
||||
}()
|
||||
log.Printf("control-plane API listening on %s", config.ListenAddress)
|
||||
if err = server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
log.Fatalf("HTTP server: %v", err)
|
||||
}
|
||||
}
|
||||
527
control-plane/backend/observability.go
Normal file
527
control-plane/backend/observability.go
Normal file
@@ -0,0 +1,527 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/subtle"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"net/http"
|
||||
"os"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
var metricSelector = regexp.MustCompile(`^([a-zA-Z_:][a-zA-Z0-9_:]*)(?:\{([^}]*)\})?$`)
|
||||
var logSelector = regexp.MustCompile(`^([a-z_][a-z0-9_]*)\s*(=|=~)\s*"([^"]*)"$`)
|
||||
|
||||
var metricColumns = map[string]string{
|
||||
"vps_control_cpu_usage_percent": "cpu",
|
||||
"vps_control_memory_usage_percent": "memory",
|
||||
"vps_control_disk_usage_percent": "disk",
|
||||
"vps_control_network_rx_bytes": "rx",
|
||||
"vps_control_network_tx_bytes": "tx",
|
||||
"vps_control_connections": "connections",
|
||||
"vps_control_bans": "bans",
|
||||
}
|
||||
|
||||
type metricQuery struct {
|
||||
Name string
|
||||
ServerID string
|
||||
ServerName string
|
||||
}
|
||||
|
||||
type logMatcher struct {
|
||||
Label string
|
||||
Exact string
|
||||
Regex *regexp.Regexp
|
||||
}
|
||||
|
||||
func readIntegrationToken(path string) (string, error) {
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read integration token metadata: %w", err)
|
||||
}
|
||||
if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 || info.Mode().Perm()&0o022 != 0 {
|
||||
return "", errors.New("integration token must be a non-writable regular file")
|
||||
}
|
||||
if info.Size() > 4096 {
|
||||
return "", errors.New("integration token file is too large")
|
||||
}
|
||||
value, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read integration token: %w", err)
|
||||
}
|
||||
token := strings.TrimSpace(string(value))
|
||||
if len(token) < 32 || len(token) > 256 || strings.IndexFunc(token, func(character rune) bool {
|
||||
return character < 0x21 || character == 0x7f
|
||||
}) >= 0 {
|
||||
return "", errors.New("integration token must contain 32-256 visible bytes")
|
||||
}
|
||||
return token, nil
|
||||
}
|
||||
|
||||
func (app *App) integration(writer http.ResponseWriter, request *http.Request) {
|
||||
if len(app.integrationTokenHash) == 0 {
|
||||
writeError(writer, http.StatusNotFound, "integration API is disabled", "not_found")
|
||||
return
|
||||
}
|
||||
if !app.integrationLimiter.Allow(app.clientIP(request), time.Now()) {
|
||||
writeError(writer, http.StatusTooManyRequests, "integration request rate exceeded", "rate_limited")
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(request.Context(), 10*time.Second)
|
||||
defer cancel()
|
||||
request = request.WithContext(ctx)
|
||||
provided := strings.TrimPrefix(request.Header.Get("Authorization"), "Bearer ")
|
||||
if len(provided) > 256 || subtle.ConstantTimeCompare(tokenHash(provided), app.integrationTokenHash) != 1 {
|
||||
writeError(writer, http.StatusUnauthorized, "integration authentication required", "unauthorized")
|
||||
return
|
||||
}
|
||||
if strings.HasPrefix(request.URL.Path, "/integrations/prometheus/") {
|
||||
app.prometheusAPI(writer, request)
|
||||
return
|
||||
}
|
||||
if strings.HasPrefix(request.URL.Path, "/integrations/loki/") {
|
||||
app.lokiAPI(writer, request)
|
||||
return
|
||||
}
|
||||
writeError(writer, http.StatusNotFound, "not found", "not_found")
|
||||
}
|
||||
|
||||
func (app *App) prometheusAPI(writer http.ResponseWriter, request *http.Request) {
|
||||
path := strings.TrimPrefix(request.URL.Path, "/integrations/prometheus")
|
||||
if request.Method != http.MethodGet && request.Method != http.MethodPost {
|
||||
writePrometheusError(writer, "bad_data", "method not allowed", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
if request.Method == http.MethodPost {
|
||||
request.Body = http.MaxBytesReader(writer, request.Body, maxRequestBody)
|
||||
}
|
||||
switch path {
|
||||
case "/-/healthy", "/-/ready":
|
||||
writeJSON(writer, 200, map[string]string{"status": "success"})
|
||||
case "/api/v1/query":
|
||||
app.prometheusQuery(writer, request, false)
|
||||
case "/api/v1/query_range":
|
||||
app.prometheusQuery(writer, request, true)
|
||||
case "/api/v1/labels":
|
||||
writeJSON(writer, 200, map[string]any{"status": "success", "data": []string{"__name__", "server_id", "server_name"}})
|
||||
case "/api/v1/label/__name__/values":
|
||||
names := make([]string, 0, len(metricColumns))
|
||||
for name := range metricColumns {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
writeJSON(writer, 200, map[string]any{"status": "success", "data": names})
|
||||
case "/api/v1/label/server_id/values", "/api/v1/label/server_name/values":
|
||||
label := strings.TrimSuffix(strings.TrimPrefix(path, "/api/v1/label/"), "/values")
|
||||
values, err := app.store.MetricLabelValues(request.Context(), label)
|
||||
if err != nil {
|
||||
writePrometheusError(writer, "internal", "query failed", 500)
|
||||
return
|
||||
}
|
||||
writeJSON(writer, 200, map[string]any{"status": "success", "data": values})
|
||||
case "/api/v1/metadata":
|
||||
metadata := map[string][]map[string]string{}
|
||||
for name := range metricColumns {
|
||||
metadata[name] = []map[string]string{{"type": "gauge", "help": "VPS Control sampled metric", "unit": ""}}
|
||||
}
|
||||
writeJSON(writer, 200, map[string]any{"status": "success", "data": metadata})
|
||||
case "/api/v1/series":
|
||||
series, err := app.store.MetricSeries(request.Context())
|
||||
if err != nil {
|
||||
writePrometheusError(writer, "internal", "query failed", 500)
|
||||
return
|
||||
}
|
||||
writeJSON(writer, 200, map[string]any{"status": "success", "data": series})
|
||||
case "/api/v1/status/buildinfo":
|
||||
writeJSON(writer, 200, map[string]any{"status": "success", "data": map[string]string{"version": "vps-control-compat-v1"}})
|
||||
default:
|
||||
writePrometheusError(writer, "bad_data", "unsupported Prometheus API operation", 404)
|
||||
}
|
||||
}
|
||||
|
||||
func (app *App) prometheusQuery(writer http.ResponseWriter, request *http.Request, ranged bool) {
|
||||
if err := request.ParseForm(); err != nil {
|
||||
writePrometheusError(writer, "bad_data", "invalid form", 400)
|
||||
return
|
||||
}
|
||||
rawQuery := request.Form.Get("query")
|
||||
if !ranged && rawQuery == "1+1" {
|
||||
at, err := parseTimestamp(request.Form.Get("time"), time.Now())
|
||||
if err != nil {
|
||||
writePrometheusError(writer, "bad_data", "invalid time", 400)
|
||||
return
|
||||
}
|
||||
writeJSON(writer, 200, map[string]any{"status": "success", "data": map[string]any{
|
||||
"resultType": "scalar", "result": []any{float64(at.UnixNano()) / 1e9, "2"},
|
||||
}})
|
||||
return
|
||||
}
|
||||
query, err := parseMetricQuery(rawQuery)
|
||||
if err != nil {
|
||||
writePrometheusError(writer, "bad_data", err.Error(), 400)
|
||||
return
|
||||
}
|
||||
timeParameter := "time"
|
||||
if ranged {
|
||||
timeParameter = "end"
|
||||
}
|
||||
end, err := parseTimestamp(request.Form.Get(timeParameter), time.Now())
|
||||
if err != nil {
|
||||
writePrometheusError(writer, "bad_data", "invalid "+timeParameter, 400)
|
||||
return
|
||||
}
|
||||
if !ranged {
|
||||
points, queryErr := app.store.InstantMetric(request.Context(), query, end)
|
||||
if queryErr != nil {
|
||||
writePrometheusError(writer, "internal", "query failed", 500)
|
||||
return
|
||||
}
|
||||
writeJSON(writer, 200, prometheusVector(query.Name, points))
|
||||
return
|
||||
}
|
||||
start, err := parseTimestamp(request.Form.Get("start"), end.Add(-time.Hour))
|
||||
if err != nil || start.After(end) || end.Sub(start) > 31*24*time.Hour {
|
||||
writePrometheusError(writer, "bad_data", "range must be positive and at most 31 days", 400)
|
||||
return
|
||||
}
|
||||
step, err := parseStep(request.Form.Get("step"))
|
||||
if err != nil {
|
||||
writePrometheusError(writer, "bad_data", "invalid step", 400)
|
||||
return
|
||||
}
|
||||
if int64(end.Sub(start)/time.Second)/int64(step) > 10_000 {
|
||||
writePrometheusError(writer, "bad_data", "range contains more than 10000 points per series", 400)
|
||||
return
|
||||
}
|
||||
points, queryErr := app.store.RangeMetric(request.Context(), query, start, end, step)
|
||||
if queryErr != nil {
|
||||
writePrometheusError(writer, "internal", "query failed", 500)
|
||||
return
|
||||
}
|
||||
writeJSON(writer, 200, prometheusMatrix(query.Name, points))
|
||||
}
|
||||
|
||||
func (app *App) lokiAPI(writer http.ResponseWriter, request *http.Request) {
|
||||
path := strings.TrimPrefix(request.URL.Path, "/integrations/loki")
|
||||
if request.Method != http.MethodGet {
|
||||
writeError(writer, 405, "method not allowed", "method_not_allowed")
|
||||
return
|
||||
}
|
||||
switch path {
|
||||
case "/ready":
|
||||
writer.WriteHeader(http.StatusOK)
|
||||
case "/loki/api/v1/query_range":
|
||||
app.lokiQueryRange(writer, request)
|
||||
case "/loki/api/v1/query":
|
||||
if request.URL.Query().Get("query") != "vector(1)+vector(1)" {
|
||||
writeError(writer, 400, "only the datasource health expression is supported for instant queries", "invalid_query")
|
||||
return
|
||||
}
|
||||
at, err := parseLokiTimestamp(request.URL.Query().Get("time"), time.Now())
|
||||
if err != nil {
|
||||
writeError(writer, 400, "invalid time", "invalid_query")
|
||||
return
|
||||
}
|
||||
writeJSON(writer, 200, map[string]any{"status": "success", "data": map[string]any{
|
||||
"resultType": "vector", "result": []any{map[string]any{
|
||||
"metric": map[string]string{}, "value": []any{float64(at.UnixNano()) / 1e9, "2"},
|
||||
}},
|
||||
}})
|
||||
case "/loki/api/v1/labels":
|
||||
writeJSON(writer, 200, map[string]any{"status": "success", "data": []string{"server_id", "server_name", "unit", "priority"}})
|
||||
case "/loki/api/v1/label/server_id/values", "/loki/api/v1/label/server_name/values", "/loki/api/v1/label/unit/values", "/loki/api/v1/label/priority/values":
|
||||
label := strings.TrimSuffix(strings.TrimPrefix(path, "/loki/api/v1/label/"), "/values")
|
||||
values, err := app.store.LogLabelValues(request.Context(), label)
|
||||
if err != nil {
|
||||
writeError(writer, 500, "query failed", "internal_error")
|
||||
return
|
||||
}
|
||||
writeJSON(writer, 200, map[string]any{"status": "success", "data": values})
|
||||
case "/loki/api/v1/series":
|
||||
series, err := app.store.LogSeries(request.Context())
|
||||
if err != nil {
|
||||
writeError(writer, 500, "query failed", "internal_error")
|
||||
return
|
||||
}
|
||||
writeJSON(writer, 200, map[string]any{"status": "success", "data": series})
|
||||
default:
|
||||
writeError(writer, 404, "unsupported Loki API operation", "not_found")
|
||||
}
|
||||
}
|
||||
|
||||
func (app *App) lokiQueryRange(writer http.ResponseWriter, request *http.Request) {
|
||||
matchers, err := parseLogQuery(request.URL.Query().Get("query"))
|
||||
if err != nil {
|
||||
writeError(writer, 400, err.Error(), "invalid_query")
|
||||
return
|
||||
}
|
||||
end, err := parseLokiTimestamp(request.URL.Query().Get("end"), time.Now())
|
||||
if err != nil {
|
||||
writeError(writer, 400, "invalid end", "invalid_query")
|
||||
return
|
||||
}
|
||||
start, err := parseLokiTimestamp(request.URL.Query().Get("start"), end.Add(-time.Hour))
|
||||
if err != nil || start.After(end) || end.Sub(start) > 31*24*time.Hour {
|
||||
writeError(writer, 400, "range must be positive and at most 31 days", "invalid_query")
|
||||
return
|
||||
}
|
||||
limit := parseInt(request.URL.Query().Get("limit"), 1000)
|
||||
if limit < 1 {
|
||||
limit = 1
|
||||
}
|
||||
if limit > 5000 {
|
||||
limit = 5000
|
||||
}
|
||||
direction := request.URL.Query().Get("direction")
|
||||
if direction == "" {
|
||||
direction = "backward"
|
||||
}
|
||||
if direction != "backward" && direction != "forward" {
|
||||
writeError(writer, 400, "invalid direction", "invalid_query")
|
||||
return
|
||||
}
|
||||
entries, queryErr := app.store.IntegrationLogs(request.Context(), start, end, 20_000, direction)
|
||||
if queryErr != nil {
|
||||
writeError(writer, 500, "query failed", "internal_error")
|
||||
return
|
||||
}
|
||||
streams := buildLokiStreams(entries, matchers, limit, direction)
|
||||
writeJSON(writer, 200, map[string]any{"status": "success", "data": map[string]any{
|
||||
"resultType": "streams", "result": streams, "stats": map[string]any{},
|
||||
}})
|
||||
}
|
||||
|
||||
func parseMetricQuery(raw string) (metricQuery, error) {
|
||||
if len(raw) > 512 {
|
||||
return metricQuery{}, errors.New("query is too long")
|
||||
}
|
||||
match := metricSelector.FindStringSubmatch(strings.TrimSpace(raw))
|
||||
if match == nil {
|
||||
return metricQuery{}, errors.New("only direct metric selectors are supported")
|
||||
}
|
||||
if _, found := metricColumns[match[1]]; !found {
|
||||
return metricQuery{}, errors.New("metric is not supported")
|
||||
}
|
||||
result := metricQuery{Name: match[1]}
|
||||
if match[2] == "" {
|
||||
return result, nil
|
||||
}
|
||||
for _, selector := range strings.Split(match[2], ",") {
|
||||
parts := strings.SplitN(strings.TrimSpace(selector), "=", 2)
|
||||
if len(parts) != 2 {
|
||||
return metricQuery{}, errors.New("invalid label selector")
|
||||
}
|
||||
value, err := strconv.Unquote(strings.TrimSpace(parts[1]))
|
||||
if err != nil || len(value) > 128 {
|
||||
return metricQuery{}, errors.New("invalid label value")
|
||||
}
|
||||
switch strings.TrimSpace(parts[0]) {
|
||||
case "server_id":
|
||||
result.ServerID = value
|
||||
case "server_name":
|
||||
result.ServerName = value
|
||||
default:
|
||||
return metricQuery{}, errors.New("unsupported label")
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func parseLogQuery(raw string) ([]logMatcher, error) {
|
||||
if len(raw) < 2 || len(raw) > 512 || raw[0] != '{' || raw[len(raw)-1] != '}' {
|
||||
return nil, errors.New("a bounded stream selector is required")
|
||||
}
|
||||
content := strings.TrimSpace(raw[1 : len(raw)-1])
|
||||
if content == "" {
|
||||
return nil, errors.New("at least one stream matcher is required")
|
||||
}
|
||||
matchers := []logMatcher{}
|
||||
for _, part := range strings.Split(content, ",") {
|
||||
match := logSelector.FindStringSubmatch(strings.TrimSpace(part))
|
||||
if match == nil || !allowedLogLabel(match[1]) || len(match[3]) > 128 {
|
||||
return nil, errors.New("unsupported stream matcher")
|
||||
}
|
||||
item := logMatcher{Label: match[1]}
|
||||
if match[2] == "=" {
|
||||
item.Exact = match[3]
|
||||
} else {
|
||||
compiled, err := regexp.Compile("^(?:" + match[3] + ")$")
|
||||
if err != nil {
|
||||
return nil, errors.New("invalid matcher regex")
|
||||
}
|
||||
item.Regex = compiled
|
||||
}
|
||||
matchers = append(matchers, item)
|
||||
}
|
||||
return matchers, nil
|
||||
}
|
||||
|
||||
func allowedLogLabel(label string) bool {
|
||||
return label == "server_id" || label == "server_name" || label == "unit" || label == "priority"
|
||||
}
|
||||
|
||||
func parseTimestamp(raw string, fallback time.Time) (time.Time, error) {
|
||||
if raw == "" {
|
||||
return fallback, nil
|
||||
}
|
||||
value, err := strconv.ParseFloat(raw, 64)
|
||||
if err != nil || math.IsNaN(value) || math.IsInf(value, 0) {
|
||||
return time.Time{}, errors.New("invalid timestamp")
|
||||
}
|
||||
seconds, fraction := math.Modf(value)
|
||||
parsed := time.Unix(int64(seconds), int64(fraction*1e9))
|
||||
if !validIntegrationTime(parsed) {
|
||||
return time.Time{}, errors.New("timestamp out of range")
|
||||
}
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
func parseLokiTimestamp(raw string, fallback time.Time) (time.Time, error) {
|
||||
if raw == "" {
|
||||
return fallback, nil
|
||||
}
|
||||
if value, err := strconv.ParseInt(raw, 10, 64); err == nil {
|
||||
var parsed time.Time
|
||||
if value > 1_000_000_000_000 {
|
||||
parsed = time.Unix(0, value)
|
||||
} else {
|
||||
parsed = time.Unix(value, 0)
|
||||
}
|
||||
if !validIntegrationTime(parsed) {
|
||||
return time.Time{}, errors.New("timestamp out of range")
|
||||
}
|
||||
return parsed, nil
|
||||
}
|
||||
parsed, err := time.Parse(time.RFC3339Nano, raw)
|
||||
if err != nil || !validIntegrationTime(parsed) {
|
||||
return time.Time{}, errors.New("invalid timestamp")
|
||||
}
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
func validIntegrationTime(value time.Time) bool {
|
||||
return !value.Before(time.Unix(0, 0)) && value.Before(time.Date(2100, 1, 1, 0, 0, 0, 0, time.UTC))
|
||||
}
|
||||
|
||||
func parseStep(raw string) (int, error) {
|
||||
if raw == "" {
|
||||
return 60, nil
|
||||
}
|
||||
if duration, err := time.ParseDuration(raw); err == nil {
|
||||
seconds := int(duration.Seconds())
|
||||
if seconds >= 5 && seconds <= 86400 {
|
||||
return seconds, nil
|
||||
}
|
||||
}
|
||||
value, err := strconv.Atoi(raw)
|
||||
if err != nil || value < 5 || value > 86400 {
|
||||
return 0, errors.New("step out of range")
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func writePrometheusError(writer http.ResponseWriter, kind, message string, status int) {
|
||||
writeJSON(writer, status, map[string]any{"status": "error", "errorType": kind, "error": message})
|
||||
}
|
||||
|
||||
func prometheusVector(name string, points []MetricPoint) map[string]any {
|
||||
result := make([]map[string]any, 0, len(points))
|
||||
for _, point := range points {
|
||||
result = append(result, map[string]any{
|
||||
"metric": metricLabels(name, point),
|
||||
"value": []any{float64(point.Timestamp.UnixNano()) / 1e9, strconv.FormatFloat(point.Value, 'f', -1, 64)},
|
||||
})
|
||||
}
|
||||
return map[string]any{"status": "success", "data": map[string]any{"resultType": "vector", "result": result}}
|
||||
}
|
||||
|
||||
func prometheusMatrix(name string, points []MetricPoint) map[string]any {
|
||||
type series struct {
|
||||
Metric map[string]string `json:"metric"`
|
||||
Values [][]any `json:"values"`
|
||||
}
|
||||
byServer := map[string]*series{}
|
||||
order := []string{}
|
||||
for _, point := range points {
|
||||
item := byServer[point.ServerID]
|
||||
if item == nil {
|
||||
item = &series{Metric: metricLabels(name, point)}
|
||||
byServer[point.ServerID] = item
|
||||
order = append(order, point.ServerID)
|
||||
}
|
||||
item.Values = append(item.Values, []any{
|
||||
float64(point.Timestamp.UnixNano()) / 1e9,
|
||||
strconv.FormatFloat(point.Value, 'f', -1, 64),
|
||||
})
|
||||
}
|
||||
result := make([]*series, 0, len(order))
|
||||
for _, id := range order {
|
||||
result = append(result, byServer[id])
|
||||
}
|
||||
return map[string]any{"status": "success", "data": map[string]any{"resultType": "matrix", "result": result}}
|
||||
}
|
||||
|
||||
func metricLabels(name string, point MetricPoint) map[string]string {
|
||||
return map[string]string{"__name__": name, "server_id": point.ServerID, "server_name": point.ServerName}
|
||||
}
|
||||
|
||||
type lokiStream struct {
|
||||
Stream map[string]string `json:"stream"`
|
||||
Values [][2]string `json:"values"`
|
||||
}
|
||||
|
||||
func buildLokiStreams(entries []IntegrationLog, matchers []logMatcher, limit int, direction string) []lokiStream {
|
||||
groups := map[string]*lokiStream{}
|
||||
count := 0
|
||||
for _, entry := range entries {
|
||||
labels := map[string]string{
|
||||
"server_id": entry.ServerID, "server_name": entry.ServerName,
|
||||
"unit": entry.Unit, "priority": strconv.Itoa(entry.Priority),
|
||||
}
|
||||
if !logMatches(labels, matchers) {
|
||||
continue
|
||||
}
|
||||
key := entry.ServerID + "\x00" + entry.Unit + "\x00" + strconv.Itoa(entry.Priority)
|
||||
stream := groups[key]
|
||||
if stream == nil {
|
||||
stream = &lokiStream{Stream: labels}
|
||||
groups[key] = stream
|
||||
}
|
||||
stream.Values = append(stream.Values, [2]string{strconv.FormatInt(entry.Timestamp.UnixNano(), 10), entry.Message})
|
||||
count++
|
||||
if count >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
result := make([]lokiStream, 0, len(groups))
|
||||
for _, stream := range groups {
|
||||
if direction != "backward" {
|
||||
sort.Slice(stream.Values, func(i, j int) bool { return stream.Values[i][0] < stream.Values[j][0] })
|
||||
}
|
||||
result = append(result, *stream)
|
||||
}
|
||||
sort.Slice(result, func(i, j int) bool {
|
||||
return result[i].Stream["server_id"]+result[i].Stream["unit"] < result[j].Stream["server_id"]+result[j].Stream["unit"]
|
||||
})
|
||||
return result
|
||||
}
|
||||
|
||||
func logMatches(labels map[string]string, matchers []logMatcher) bool {
|
||||
for _, matcher := range matchers {
|
||||
value := labels[matcher.Label]
|
||||
if matcher.Regex != nil {
|
||||
if !matcher.Regex.MatchString(value) {
|
||||
return false
|
||||
}
|
||||
} else if value != matcher.Exact {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
101
control-plane/backend/observability_test.go
Normal file
101
control-plane/backend/observability_test.go
Normal file
@@ -0,0 +1,101 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestParseMetricQuery(t *testing.T) {
|
||||
query, err := parseMetricQuery(`vps_control_cpu_usage_percent{server_id="srv-1",server_name="edge"}`)
|
||||
if err != nil || query.Name != "vps_control_cpu_usage_percent" || query.ServerID != "srv-1" || query.ServerName != "edge" {
|
||||
t.Fatalf("unexpected metric query: %#v, %v", query, err)
|
||||
}
|
||||
for _, invalid := range []string{
|
||||
`process_cpu_seconds_total`,
|
||||
`vps_control_cpu_usage_percent{job=~".*"}`,
|
||||
`vps_control_cpu_usage_percent or vector(1)`,
|
||||
} {
|
||||
if _, err = parseMetricQuery(invalid); err == nil {
|
||||
t.Fatalf("accepted unsupported PromQL %q", invalid)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseLogQueryIsAllowListedAndAnchored(t *testing.T) {
|
||||
matchers, err := parseLogQuery(`{server_name="edge",unit=~"(nginx|ssh)\.service"}`)
|
||||
if err != nil || len(matchers) != 2 {
|
||||
t.Fatalf("unexpected log selector: %#v, %v", matchers, err)
|
||||
}
|
||||
if logMatches(map[string]string{"server_name": "edge", "unit": "prefix-nginx.service"}, matchers) {
|
||||
t.Fatal("regex matcher was not anchored")
|
||||
}
|
||||
if !logMatches(map[string]string{"server_name": "edge", "unit": "nginx.service"}, matchers) {
|
||||
t.Fatal("valid labels did not match")
|
||||
}
|
||||
for _, invalid := range []string{`{}`, `{message="secret"}`, `{unit!="ssh.service"}`, `{unit="x"} |= "error"`} {
|
||||
if _, err = parseLogQuery(invalid); err == nil {
|
||||
t.Fatalf("accepted unsupported LogQL %q", invalid)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildLokiStreamsIsBounded(t *testing.T) {
|
||||
now := time.Now()
|
||||
entries := []IntegrationLog{
|
||||
{ServerID: "1", ServerName: "edge", Unit: "nginx.service", Priority: 6, Timestamp: now, Message: "new"},
|
||||
{ServerID: "1", ServerName: "edge", Unit: "nginx.service", Priority: 6, Timestamp: now.Add(-time.Second), Message: "old"},
|
||||
}
|
||||
streams := buildLokiStreams(entries, []logMatcher{{Label: "server_id", Exact: "1"}}, 1, "backward")
|
||||
if len(streams) != 1 || len(streams[0].Values) != 1 || streams[0].Values[0][1] != "new" {
|
||||
t.Fatalf("unexpected bounded streams: %#v", streams)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadIntegrationTokenPermissionsAndLength(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "token")
|
||||
if err := os.WriteFile(path, []byte("0123456789abcdef0123456789abcdef\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := readIntegrationToken(path); err != nil {
|
||||
t.Fatalf("valid token rejected: %v", err)
|
||||
}
|
||||
if err := os.Chmod(path, 0o622); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := readIntegrationToken(path); err == nil {
|
||||
t.Fatal("group-writable token accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTimestampBounds(t *testing.T) {
|
||||
if _, err := parseTimestamp("NaN", time.Now()); err == nil {
|
||||
t.Fatal("NaN timestamp accepted")
|
||||
}
|
||||
if _, err := parseLokiTimestamp("4102444800000000000", time.Now()); err == nil {
|
||||
t.Fatal("2100 timestamp accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGrafanaDatasourceHealthExpressions(t *testing.T) {
|
||||
app := &App{}
|
||||
prometheusRequest := httptest.NewRequest(http.MethodPost, "/", strings.NewReader("query=1%2B1&time=4"))
|
||||
prometheusRequest.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
prometheusResponse := httptest.NewRecorder()
|
||||
app.prometheusQuery(prometheusResponse, prometheusRequest, false)
|
||||
if prometheusResponse.Code != http.StatusOK || !strings.Contains(prometheusResponse.Body.String(), `"resultType":"scalar"`) {
|
||||
t.Fatalf("unexpected Prometheus health response: %d %s", prometheusResponse.Code, prometheusResponse.Body.String())
|
||||
}
|
||||
|
||||
lokiRequest := httptest.NewRequest(http.MethodGet,
|
||||
"/integrations/loki/loki/api/v1/query?query=vector%281%29%2Bvector%281%29&time=4000000000", nil)
|
||||
lokiResponse := httptest.NewRecorder()
|
||||
app.lokiAPI(lokiResponse, lokiRequest)
|
||||
if lokiResponse.Code != http.StatusOK || !strings.Contains(lokiResponse.Body.String(), `"resultType":"vector"`) {
|
||||
t.Fatalf("unexpected Loki health response: %d %s", lokiResponse.Code, lokiResponse.Body.String())
|
||||
}
|
||||
}
|
||||
254
control-plane/backend/poller.go
Normal file
254
control-plane/backend/poller.go
Normal file
@@ -0,0 +1,254 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
var bearerSecret = regexp.MustCompile(`(?i)(authorization\s*[:=]\s*bearer\s+)[^\s,;&]+`)
|
||||
var namedSecret = regexp.MustCompile(`(?i)((?:password|passwd|token|secret|api[_-]?key)["']?\s*[:=]\s*)(?:"[^"]*"|'[^']*'|[^\s,;&]+)`)
|
||||
|
||||
type Poller struct {
|
||||
store *Store
|
||||
agent *AgentClient
|
||||
metricRetention time.Duration
|
||||
logRetention time.Duration
|
||||
}
|
||||
|
||||
func (poller *Poller) Run(ctx context.Context) {
|
||||
pollTicker := time.NewTicker(time.Minute)
|
||||
cleanupTicker := time.NewTicker(time.Hour)
|
||||
defer pollTicker.Stop()
|
||||
defer cleanupTicker.Stop()
|
||||
poller.poll(ctx)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-pollTicker.C:
|
||||
poller.poll(ctx)
|
||||
case <-cleanupTicker.C:
|
||||
if _, err := poller.store.pool.Exec(ctx, "DELETE FROM metric_samples WHERE sampled_at < $1", time.Now().Add(-poller.metricRetention)); err != nil {
|
||||
log.Printf("metric retention cleanup: %v", err)
|
||||
}
|
||||
if _, err := poller.store.pool.Exec(ctx, "DELETE FROM agent_log_entries WHERE event_at < $1", time.Now().Add(-poller.logRetention)); err != nil {
|
||||
log.Printf("log retention cleanup: %v", err)
|
||||
}
|
||||
_, _ = poller.store.pool.Exec(ctx, "DELETE FROM sessions WHERE expires_at < now()")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (poller *Poller) poll(ctx context.Context) {
|
||||
servers, err := poller.store.Servers(ctx)
|
||||
if err != nil {
|
||||
log.Printf("poll server list: %v", err)
|
||||
return
|
||||
}
|
||||
jobs := make(chan Server)
|
||||
var wait sync.WaitGroup
|
||||
workerCount := 4
|
||||
if len(servers) < workerCount {
|
||||
workerCount = len(servers)
|
||||
}
|
||||
for worker := 0; worker < workerCount; worker++ {
|
||||
wait.Add(1)
|
||||
go func() {
|
||||
defer wait.Done()
|
||||
for server := range jobs {
|
||||
poller.pollServer(ctx, server)
|
||||
}
|
||||
}()
|
||||
}
|
||||
for _, server := range servers {
|
||||
select {
|
||||
case jobs <- server:
|
||||
case <-ctx.Done():
|
||||
close(jobs)
|
||||
wait.Wait()
|
||||
return
|
||||
}
|
||||
}
|
||||
close(jobs)
|
||||
wait.Wait()
|
||||
}
|
||||
|
||||
func (poller *Poller) pollServer(ctx context.Context, server Server) {
|
||||
statusCode, body, err := poller.agent.Do(ctx, server, http.MethodGet, "/v1/system/status", nil)
|
||||
if err != nil || statusCode != http.StatusOK {
|
||||
_, _ = poller.store.pool.Exec(ctx, "UPDATE servers SET status='offline',updated_at=now() WHERE id=$1", server.ID)
|
||||
return
|
||||
}
|
||||
var status struct {
|
||||
Hostname string `json:"hostname"`
|
||||
CPU float64 `json:"cpu_usage_percent"`
|
||||
MemoryTotal uint64 `json:"memory_total_bytes"`
|
||||
MemoryUsed uint64 `json:"memory_used_bytes"`
|
||||
Filesystems []struct {
|
||||
Total uint64 `json:"total_bytes"`
|
||||
Used uint64 `json:"used_bytes"`
|
||||
} `json:"filesystems"`
|
||||
}
|
||||
if json.Unmarshal(body, &status) != nil {
|
||||
return
|
||||
}
|
||||
memory := percent(status.MemoryUsed, status.MemoryTotal)
|
||||
var diskTotal, diskUsed uint64
|
||||
for _, filesystem := range status.Filesystems {
|
||||
diskTotal += filesystem.Total
|
||||
diskUsed += filesystem.Used
|
||||
}
|
||||
disk := percent(diskUsed, diskTotal)
|
||||
var rx, tx int64
|
||||
if code, payload, callErr := poller.agent.Do(ctx, server, http.MethodGet, "/v1/network/interfaces", nil); callErr == nil && code == 200 {
|
||||
var interfaces []struct {
|
||||
RX int64 `json:"rx_bytes"`
|
||||
TX int64 `json:"tx_bytes"`
|
||||
}
|
||||
if json.Unmarshal(payload, &interfaces) == nil {
|
||||
for _, item := range interfaces {
|
||||
rx += item.RX
|
||||
tx += item.TX
|
||||
}
|
||||
}
|
||||
}
|
||||
connections := 0
|
||||
if code, payload, callErr := poller.agent.Do(ctx, server, http.MethodGet, "/v1/network/connections", nil); callErr == nil && code == 200 {
|
||||
var items []json.RawMessage
|
||||
if json.Unmarshal(payload, &items) == nil {
|
||||
connections = len(items)
|
||||
}
|
||||
}
|
||||
txDB, err := poller.store.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer txDB.Rollback(ctx)
|
||||
_, err = txDB.Exec(ctx, "UPDATE servers SET status='online',hostname=$2,last_seen=now(),updated_at=now() WHERE id=$1", server.ID, status.Hostname)
|
||||
if err == nil {
|
||||
_, err = txDB.Exec(ctx, `INSERT INTO metric_samples(server_id,cpu,memory,disk,rx,tx,connections,bans)
|
||||
VALUES($1,$2,$3,$4,$5,$6,$7,0)`, server.ID, status.CPU, memory, disk, rx, tx, connections)
|
||||
}
|
||||
if err == nil {
|
||||
err = txDB.Commit(ctx)
|
||||
}
|
||||
if err != nil {
|
||||
log.Printf("store metric for %s: %v", server.ID, err)
|
||||
}
|
||||
poller.pollLogs(ctx, server)
|
||||
}
|
||||
|
||||
func (poller *Poller) pollLogs(ctx context.Context, server Server) {
|
||||
var cursor string
|
||||
var since time.Time
|
||||
err := poller.store.pool.QueryRow(ctx, `SELECT
|
||||
COALESCE((SELECT journal_cursor FROM agent_log_entries WHERE server_id=$1 ORDER BY event_at DESC,ingested_at DESC LIMIT 1),''),
|
||||
COALESCE((SELECT max(event_at) FROM agent_log_entries WHERE server_id=$1),now() - interval '2 minutes')`, server.ID).
|
||||
Scan(&cursor, &since)
|
||||
if err != nil {
|
||||
log.Printf("log cursor for %s: %v", server.ID, err)
|
||||
return
|
||||
}
|
||||
fallbackUsed := false
|
||||
for page := 0; page < 5; page++ {
|
||||
path := fmt.Sprintf("/v1/logs?since=%d&limit=200", since.Add(-time.Second).Unix())
|
||||
if cursor != "" {
|
||||
path = "/v1/logs?cursor=" + url.QueryEscape(cursor) + "&limit=200"
|
||||
}
|
||||
status, payload, callErr := poller.agent.Do(ctx, server, http.MethodGet, path, nil)
|
||||
if callErr != nil || status == http.StatusForbidden || status == http.StatusNotFound {
|
||||
return
|
||||
}
|
||||
if status != http.StatusOK {
|
||||
if cursor != "" && !fallbackUsed {
|
||||
cursor = ""
|
||||
fallbackUsed = true
|
||||
page--
|
||||
continue
|
||||
}
|
||||
log.Printf("log poll for %s returned %d", server.ID, status)
|
||||
return
|
||||
}
|
||||
var response struct {
|
||||
Entries []struct {
|
||||
Cursor string `json:"cursor"`
|
||||
Timestamp uint64 `json:"timestamp_unix_micros"`
|
||||
Unit string `json:"unit"`
|
||||
Priority int `json:"priority"`
|
||||
Message string `json:"message"`
|
||||
BootID *string `json:"boot_id"`
|
||||
PID *int `json:"pid"`
|
||||
} `json:"entries"`
|
||||
Truncated bool `json:"truncated"`
|
||||
}
|
||||
if err = json.Unmarshal(payload, &response); err != nil {
|
||||
log.Printf("decode logs for %s: %v", server.ID, err)
|
||||
return
|
||||
}
|
||||
if len(response.Entries) > 200 {
|
||||
log.Printf("agent %s exceeded log entry bound", server.ID)
|
||||
return
|
||||
}
|
||||
tx, beginErr := poller.store.pool.Begin(ctx)
|
||||
if beginErr != nil {
|
||||
return
|
||||
}
|
||||
now := time.Now()
|
||||
for _, entry := range response.Entries {
|
||||
if len(entry.Cursor) == 0 || len(entry.Cursor) > 1024 || len(entry.Unit) > 128 ||
|
||||
len(entry.Message) > 65_536 || entry.Priority < 0 || entry.Priority > 7 {
|
||||
continue
|
||||
}
|
||||
eventAt := time.UnixMicro(int64(entry.Timestamp))
|
||||
if eventAt.After(now.Add(5*time.Minute)) || eventAt.Before(now.Add(-31*24*time.Hour)) {
|
||||
continue
|
||||
}
|
||||
_, err = tx.Exec(ctx, `INSERT INTO agent_log_entries
|
||||
(server_id,journal_cursor,event_at,unit,priority,message,boot_id,pid)
|
||||
VALUES($1,$2,$3,$4,$5,$6,$7,$8) ON CONFLICT DO NOTHING`, server.ID, entry.Cursor,
|
||||
eventAt, entry.Unit, entry.Priority, truncateUTF8(redactLogMessage(entry.Message), 65_536), entry.BootID, entry.PID)
|
||||
if err != nil {
|
||||
_ = tx.Rollback(ctx)
|
||||
return
|
||||
}
|
||||
}
|
||||
if err = tx.Commit(ctx); err != nil {
|
||||
log.Printf("store logs for %s: %v", server.ID, err)
|
||||
return
|
||||
}
|
||||
if len(response.Entries) == 0 || !response.Truncated {
|
||||
return
|
||||
}
|
||||
cursor = response.Entries[len(response.Entries)-1].Cursor
|
||||
}
|
||||
log.Printf("agent log backlog for %s exceeded 1000 entries in one poll", server.ID)
|
||||
}
|
||||
|
||||
func redactLogMessage(message string) string {
|
||||
message = bearerSecret.ReplaceAllString(message, "${1}[REDACTED]")
|
||||
return namedSecret.ReplaceAllString(message, "${1}[REDACTED]")
|
||||
}
|
||||
|
||||
func truncateUTF8(value string, maximum int) string {
|
||||
if len(value) <= maximum {
|
||||
return value
|
||||
}
|
||||
for maximum > 0 && value[maximum]&0xc0 == 0x80 {
|
||||
maximum--
|
||||
}
|
||||
return value[:maximum]
|
||||
}
|
||||
|
||||
func percent(used, total uint64) float64 {
|
||||
if total == 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(used) * 100 / float64(total)
|
||||
}
|
||||
Reference in New Issue
Block a user