377 lines
12 KiB
Go
377 lines
12 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"log"
|
|
"net"
|
|
"net/http"
|
|
"net/url"
|
|
"regexp"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
var bearerSecret = regexp.MustCompile(`(?i)(authorization\s*[:=]\s*bearer\s+)[^\s,;&]+`)
|
|
var namedSecret = regexp.MustCompile(`(?i)((?:password|passwd|token|secret|api[_-]?key)["']?\s*[:=]\s*)(?:"[^"]*"|'[^']*'|[^\s,;&]+)`)
|
|
|
|
type Poller struct {
|
|
store *Store
|
|
agent *AgentClient
|
|
metricRetention time.Duration
|
|
logRetention time.Duration
|
|
}
|
|
|
|
func (poller *Poller) Run(ctx context.Context) {
|
|
pollTicker := time.NewTicker(time.Minute)
|
|
cleanupTicker := time.NewTicker(time.Hour)
|
|
defer pollTicker.Stop()
|
|
defer cleanupTicker.Stop()
|
|
poller.poll(ctx)
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-pollTicker.C:
|
|
poller.poll(ctx)
|
|
case <-cleanupTicker.C:
|
|
if _, err := poller.store.pool.Exec(ctx, "DELETE FROM metric_samples WHERE sampled_at < $1", time.Now().Add(-poller.metricRetention)); err != nil {
|
|
log.Printf("metric retention cleanup: %v", err)
|
|
}
|
|
if _, err := poller.store.pool.Exec(ctx, "DELETE FROM agent_log_entries WHERE event_at < $1", time.Now().Add(-poller.logRetention)); err != nil {
|
|
log.Printf("log retention cleanup: %v", err)
|
|
}
|
|
_, _ = poller.store.pool.Exec(ctx, "DELETE FROM connection_samples WHERE sampled_at < $1", time.Now().Add(-poller.metricRetention))
|
|
_, _ = poller.store.pool.Exec(ctx, "DELETE FROM fail2ban_events WHERE event_at < $1", time.Now().Add(-poller.logRetention))
|
|
_, _ = poller.store.pool.Exec(ctx, "DELETE FROM sessions WHERE expires_at < now()")
|
|
}
|
|
}
|
|
}
|
|
|
|
func (poller *Poller) poll(ctx context.Context) {
|
|
servers, err := poller.store.Servers(ctx)
|
|
if err != nil {
|
|
log.Printf("poll server list: %v", err)
|
|
return
|
|
}
|
|
jobs := make(chan Server)
|
|
var wait sync.WaitGroup
|
|
workerCount := 4
|
|
if len(servers) < workerCount {
|
|
workerCount = len(servers)
|
|
}
|
|
for worker := 0; worker < workerCount; worker++ {
|
|
wait.Add(1)
|
|
go func() {
|
|
defer wait.Done()
|
|
for server := range jobs {
|
|
poller.pollServer(ctx, server)
|
|
}
|
|
}()
|
|
}
|
|
for _, server := range servers {
|
|
select {
|
|
case jobs <- server:
|
|
case <-ctx.Done():
|
|
close(jobs)
|
|
wait.Wait()
|
|
return
|
|
}
|
|
}
|
|
close(jobs)
|
|
wait.Wait()
|
|
}
|
|
|
|
func (poller *Poller) pollServer(ctx context.Context, server Server) {
|
|
statusCode, body, err := poller.agent.Do(ctx, server, http.MethodGet, "/v1/system/status", nil)
|
|
if err != nil || statusCode != http.StatusOK {
|
|
_, _ = poller.store.pool.Exec(ctx, "UPDATE servers SET status='offline',updated_at=now() WHERE id=$1", server.ID)
|
|
return
|
|
}
|
|
var status struct {
|
|
Hostname string `json:"hostname"`
|
|
CPU float64 `json:"cpu_usage_percent"`
|
|
MemoryTotal uint64 `json:"memory_total_bytes"`
|
|
MemoryUsed uint64 `json:"memory_used_bytes"`
|
|
Filesystems []struct {
|
|
Total uint64 `json:"total_bytes"`
|
|
Used uint64 `json:"used_bytes"`
|
|
} `json:"filesystems"`
|
|
}
|
|
if json.Unmarshal(body, &status) != nil {
|
|
return
|
|
}
|
|
memory := percent(status.MemoryUsed, status.MemoryTotal)
|
|
var diskTotal, diskUsed uint64
|
|
for _, filesystem := range status.Filesystems {
|
|
diskTotal += filesystem.Total
|
|
diskUsed += filesystem.Used
|
|
}
|
|
disk := percent(diskUsed, diskTotal)
|
|
var rx, tx int64
|
|
if code, payload, callErr := poller.agent.Do(ctx, server, http.MethodGet, "/v1/network/interfaces", nil); callErr == nil && code == 200 {
|
|
var interfaces []struct {
|
|
RX int64 `json:"rx_bytes"`
|
|
TX int64 `json:"tx_bytes"`
|
|
}
|
|
if json.Unmarshal(payload, &interfaces) == nil {
|
|
for _, item := range interfaces {
|
|
rx += item.RX
|
|
tx += item.TX
|
|
}
|
|
}
|
|
}
|
|
type socket struct {
|
|
Protocol string `json:"protocol"`
|
|
LocalAddress string `json:"local_address"`
|
|
LocalPort int `json:"local_port"`
|
|
RemoteAddress string `json:"remote_address"`
|
|
RemotePort int `json:"remote_port"`
|
|
State string `json:"state"`
|
|
Process *string `json:"process"`
|
|
}
|
|
connections := 0
|
|
connectionRows := []socket{}
|
|
if code, payload, callErr := poller.agent.Do(ctx, server, http.MethodGet, "/v1/network/connections", nil); callErr == nil && code == 200 {
|
|
if json.Unmarshal(payload, &connectionRows) == nil {
|
|
if len(connectionRows) > 10000 {
|
|
connectionRows = connectionRows[:10000]
|
|
}
|
|
connections = len(connectionRows)
|
|
}
|
|
}
|
|
bans := poller.pollFail2Ban(ctx, server)
|
|
txDB, err := poller.store.pool.Begin(ctx)
|
|
if err != nil {
|
|
return
|
|
}
|
|
defer txDB.Rollback(ctx)
|
|
_, err = txDB.Exec(ctx, "UPDATE servers SET status='online',hostname=$2,last_seen=now(),updated_at=now() WHERE id=$1", server.ID, status.Hostname)
|
|
if err == nil {
|
|
_, err = txDB.Exec(ctx, `INSERT INTO metric_samples(server_id,cpu,memory,disk,rx,tx,connections,bans)
|
|
VALUES($1,$2,$3,$4,$5,$6,$7,$8)`, server.ID, status.CPU, memory, disk, rx, tx, connections, bans)
|
|
}
|
|
if err == nil {
|
|
type aggregate struct {
|
|
row socket
|
|
count int
|
|
}
|
|
grouped := map[string]aggregate{}
|
|
for _, row := range connectionRows {
|
|
if net.ParseIP(row.LocalAddress) == nil || net.ParseIP(row.RemoteAddress) == nil || row.LocalPort < 0 || row.LocalPort > 65535 || row.RemotePort < 0 || row.RemotePort > 65535 || len(row.Protocol) > 8 || len(row.State) > 32 {
|
|
continue
|
|
}
|
|
process := ""
|
|
if row.Process != nil {
|
|
process = truncateUTF8(*row.Process, 128)
|
|
}
|
|
key := fmt.Sprintf("%s\x00%s\x00%d\x00%s\x00%d\x00%s\x00%s", row.Protocol, row.LocalAddress, row.LocalPort, row.RemoteAddress, row.RemotePort, row.State, process)
|
|
entry := grouped[key]
|
|
entry.row = row
|
|
entry.row.Process = &process
|
|
entry.count++
|
|
grouped[key] = entry
|
|
}
|
|
inserted := 0
|
|
for _, entry := range grouped {
|
|
if inserted >= 2000 {
|
|
break
|
|
}
|
|
_, err = txDB.Exec(ctx, `INSERT INTO connection_samples(server_id,protocol,local_address,local_port,remote_address,remote_port,state,process,connection_count)
|
|
VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9)`, server.ID, entry.row.Protocol, entry.row.LocalAddress, entry.row.LocalPort, entry.row.RemoteAddress, entry.row.RemotePort, entry.row.State, *entry.row.Process, entry.count)
|
|
if err != nil {
|
|
break
|
|
}
|
|
inserted++
|
|
}
|
|
}
|
|
if err == nil {
|
|
err = txDB.Commit(ctx)
|
|
}
|
|
if err != nil {
|
|
log.Printf("store metric for %s: %v", server.ID, err)
|
|
}
|
|
poller.pollLogs(ctx, server)
|
|
}
|
|
|
|
func (poller *Poller) pollFail2Ban(ctx context.Context, server Server) int {
|
|
status, payload, err := poller.agent.Do(ctx, server, http.MethodGet, "/v1/fail2ban/status", nil)
|
|
if err != nil || status != http.StatusOK {
|
|
return 0
|
|
}
|
|
var response struct {
|
|
JailStatuses []struct {
|
|
Jail string `json:"jail"`
|
|
Banned []string `json:"banned_ips"`
|
|
} `json:"jail_statuses"`
|
|
}
|
|
if json.Unmarshal(payload, &response) != nil || len(response.JailStatuses) > 64 {
|
|
return 0
|
|
}
|
|
current := map[string][2]string{}
|
|
total := 0
|
|
for _, jail := range response.JailStatuses {
|
|
if !validSimpleName(jail.Jail) || len(jail.Banned) > 10000 {
|
|
continue
|
|
}
|
|
for _, ip := range jail.Banned {
|
|
if net.ParseIP(ip) != nil {
|
|
current[jail.Jail+"\x00"+ip] = [2]string{jail.Jail, ip}
|
|
total++
|
|
}
|
|
}
|
|
}
|
|
rows, err := poller.store.pool.Query(ctx, "SELECT jail,ip FROM fail2ban_ban_state WHERE server_id=$1", server.ID)
|
|
if err != nil {
|
|
return total
|
|
}
|
|
previous := map[string][2]string{}
|
|
for rows.Next() {
|
|
var jail, ip string
|
|
if rows.Scan(&jail, &ip) == nil {
|
|
previous[jail+"\x00"+ip] = [2]string{jail, ip}
|
|
}
|
|
}
|
|
rows.Close()
|
|
tx, err := poller.store.pool.Begin(ctx)
|
|
if err != nil {
|
|
return total
|
|
}
|
|
defer tx.Rollback(ctx)
|
|
for key, value := range current {
|
|
if _, found := previous[key]; !found {
|
|
_, err = tx.Exec(ctx, "INSERT INTO fail2ban_events(server_id,jail,ip,action,source) VALUES($1,$2,$3,'ban','agent')", server.ID, value[0], value[1])
|
|
if err != nil {
|
|
return total
|
|
}
|
|
}
|
|
}
|
|
for key, value := range previous {
|
|
if _, found := current[key]; !found {
|
|
_, err = tx.Exec(ctx, "INSERT INTO fail2ban_events(server_id,jail,ip,action,source) VALUES($1,$2,$3,'unban','agent')", server.ID, value[0], value[1])
|
|
if err != nil {
|
|
return total
|
|
}
|
|
}
|
|
}
|
|
if _, err = tx.Exec(ctx, "DELETE FROM fail2ban_ban_state WHERE server_id=$1", server.ID); err != nil {
|
|
return total
|
|
}
|
|
for _, value := range current {
|
|
if _, err = tx.Exec(ctx, "INSERT INTO fail2ban_ban_state(server_id,jail,ip) VALUES($1,$2,$3)", server.ID, value[0], value[1]); err != nil {
|
|
return total
|
|
}
|
|
}
|
|
_ = tx.Commit(ctx)
|
|
return total
|
|
}
|
|
|
|
func (poller *Poller) pollLogs(ctx context.Context, server Server) {
|
|
var cursor string
|
|
var since time.Time
|
|
err := poller.store.pool.QueryRow(ctx, `SELECT
|
|
COALESCE((SELECT journal_cursor FROM agent_log_entries WHERE server_id=$1 ORDER BY event_at DESC,ingested_at DESC LIMIT 1),''),
|
|
COALESCE((SELECT max(event_at) FROM agent_log_entries WHERE server_id=$1),now() - interval '2 minutes')`, server.ID).
|
|
Scan(&cursor, &since)
|
|
if err != nil {
|
|
log.Printf("log cursor for %s: %v", server.ID, err)
|
|
return
|
|
}
|
|
fallbackUsed := false
|
|
for page := 0; page < 5; page++ {
|
|
path := fmt.Sprintf("/v1/logs?since=%d&limit=200", since.Add(-time.Second).Unix())
|
|
if cursor != "" {
|
|
path = "/v1/logs?cursor=" + url.QueryEscape(cursor) + "&limit=200"
|
|
}
|
|
status, payload, callErr := poller.agent.Do(ctx, server, http.MethodGet, path, nil)
|
|
if callErr != nil || status == http.StatusForbidden || status == http.StatusNotFound {
|
|
return
|
|
}
|
|
if status != http.StatusOK {
|
|
if cursor != "" && !fallbackUsed {
|
|
cursor = ""
|
|
fallbackUsed = true
|
|
page--
|
|
continue
|
|
}
|
|
log.Printf("log poll for %s returned %d", server.ID, status)
|
|
return
|
|
}
|
|
var response struct {
|
|
Entries []struct {
|
|
Cursor string `json:"cursor"`
|
|
Timestamp uint64 `json:"timestamp_unix_micros"`
|
|
Unit string `json:"unit"`
|
|
Priority int `json:"priority"`
|
|
Message string `json:"message"`
|
|
BootID *string `json:"boot_id"`
|
|
PID *int `json:"pid"`
|
|
} `json:"entries"`
|
|
Truncated bool `json:"truncated"`
|
|
}
|
|
if err = json.Unmarshal(payload, &response); err != nil {
|
|
log.Printf("decode logs for %s: %v", server.ID, err)
|
|
return
|
|
}
|
|
if len(response.Entries) > 200 {
|
|
log.Printf("agent %s exceeded log entry bound", server.ID)
|
|
return
|
|
}
|
|
tx, beginErr := poller.store.pool.Begin(ctx)
|
|
if beginErr != nil {
|
|
return
|
|
}
|
|
now := time.Now()
|
|
for _, entry := range response.Entries {
|
|
if len(entry.Cursor) == 0 || len(entry.Cursor) > 1024 || len(entry.Unit) > 128 ||
|
|
len(entry.Message) > 65_536 || entry.Priority < 0 || entry.Priority > 7 {
|
|
continue
|
|
}
|
|
eventAt := time.UnixMicro(int64(entry.Timestamp))
|
|
if eventAt.After(now.Add(5*time.Minute)) || eventAt.Before(now.Add(-31*24*time.Hour)) {
|
|
continue
|
|
}
|
|
_, err = tx.Exec(ctx, `INSERT INTO agent_log_entries
|
|
(server_id,journal_cursor,event_at,unit,priority,message,boot_id,pid)
|
|
VALUES($1,$2,$3,$4,$5,$6,$7,$8) ON CONFLICT DO NOTHING`, server.ID, entry.Cursor,
|
|
eventAt, entry.Unit, entry.Priority, truncateUTF8(redactLogMessage(entry.Message), 65_536), entry.BootID, entry.PID)
|
|
if err != nil {
|
|
_ = tx.Rollback(ctx)
|
|
return
|
|
}
|
|
}
|
|
if err = tx.Commit(ctx); err != nil {
|
|
log.Printf("store logs for %s: %v", server.ID, err)
|
|
return
|
|
}
|
|
if len(response.Entries) == 0 || !response.Truncated {
|
|
return
|
|
}
|
|
cursor = response.Entries[len(response.Entries)-1].Cursor
|
|
}
|
|
log.Printf("agent log backlog for %s exceeded 1000 entries in one poll", server.ID)
|
|
}
|
|
|
|
func redactLogMessage(message string) string {
|
|
message = bearerSecret.ReplaceAllString(message, "${1}[REDACTED]")
|
|
return namedSecret.ReplaceAllString(message, "${1}[REDACTED]")
|
|
}
|
|
|
|
func truncateUTF8(value string, maximum int) string {
|
|
if len(value) <= maximum {
|
|
return value
|
|
}
|
|
for maximum > 0 && value[maximum]&0xc0 == 0x80 {
|
|
maximum--
|
|
}
|
|
return value[:maximum]
|
|
}
|
|
|
|
func percent(used, total uint64) float64 {
|
|
if total == 0 {
|
|
return 0
|
|
}
|
|
return float64(used) * 100 / float64(total)
|
|
}
|