update
This commit is contained in:
@@ -127,6 +127,9 @@ func (client *AgentClient) Do(ctx context.Context, server Server, method, path s
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
if method != http.MethodGet && method != http.MethodHead {
|
||||
request.Close = true
|
||||
}
|
||||
request.Header.Set("Authorization", "Bearer "+token)
|
||||
request.Header.Set("Accept", "application/json")
|
||||
if len(body) > 0 {
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"crypto/subtle"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
@@ -275,6 +276,7 @@ func (app *App) serverRoute(writer http.ResponseWriter, request *http.Request, u
|
||||
"listeners": "/v1/network/listeners", "connections": "/v1/network/connections",
|
||||
"firewall": "/v1/firewall/rules", "fail2ban": "/v1/fail2ban/status",
|
||||
"wireguard": "/v1/wireguard/status", "services": "/v1/services",
|
||||
"rate-limits": "/v1/rate-limits",
|
||||
}
|
||||
if len(parts) == 2 && request.Method == http.MethodGet {
|
||||
if parts[1] == "metrics" {
|
||||
@@ -295,6 +297,25 @@ func (app *App) serverRoute(writer http.ResponseWriter, request *http.Request, u
|
||||
writeJSON(writer, 200, entries)
|
||||
return
|
||||
}
|
||||
if parts[1] == "connection-history" {
|
||||
hours := queryHours(request, 24, 24*30)
|
||||
history, historyErr := app.store.ConnectionHistory(request.Context(), server.ID, time.Now().Add(-time.Duration(hours)*time.Hour))
|
||||
if historyErr != nil {
|
||||
writeInternal(writer, historyErr)
|
||||
return
|
||||
}
|
||||
writeJSON(writer, 200, history)
|
||||
return
|
||||
}
|
||||
if parts[1] == "fail2ban-history" {
|
||||
events, eventErr := app.store.Fail2BanEvents(request.Context(), server.ID, queryLimit(request, 200, 1000))
|
||||
if eventErr != nil {
|
||||
writeInternal(writer, eventErr)
|
||||
return
|
||||
}
|
||||
writeJSON(writer, 200, events)
|
||||
return
|
||||
}
|
||||
if agentPath, found := readRoutes[parts[1]]; found {
|
||||
app.proxyRead(writer, request, server, agentPath)
|
||||
return
|
||||
@@ -311,11 +332,51 @@ func (app *App) serverRoute(writer http.ResponseWriter, request *http.Request, u
|
||||
app.fail2banMutation(writer, request, user, server, parts[2])
|
||||
return
|
||||
}
|
||||
if len(parts) == 4 && request.Method == http.MethodPost && parts[1] == "services" && parts[3] == "restart" && validServiceName(parts[2]) {
|
||||
if !app.authorize(writer, request, user, csrfHash, RoleOperator) {
|
||||
if len(parts) == 3 && request.Method == http.MethodPost && parts[1] == "fail2ban" && parts[2] == "configure" {
|
||||
if !app.authorize(writer, request, user, csrfHash, RoleAdministrator) {
|
||||
return
|
||||
}
|
||||
app.serviceMutation(writer, request, user, server, parts[2])
|
||||
app.fail2banConfigure(writer, request, user, server)
|
||||
return
|
||||
}
|
||||
if len(parts) == 4 && request.Method == http.MethodPost && parts[1] == "services" && validServiceName(parts[2]) {
|
||||
operation := parts[3]
|
||||
requiredRole := RoleOperator
|
||||
if operation == "install" {
|
||||
requiredRole = RoleAdministrator
|
||||
}
|
||||
if !app.authorize(writer, request, user, csrfHash, requiredRole) {
|
||||
return
|
||||
}
|
||||
app.serviceMutation(writer, request, user, server, parts[2], operation)
|
||||
return
|
||||
}
|
||||
if len(parts) == 3 && parts[1] == "firewall" && parts[2] == "rules" && request.Method == http.MethodPost {
|
||||
if !app.authorize(writer, request, user, csrfHash, RoleAdministrator) {
|
||||
return
|
||||
}
|
||||
app.firewallAdd(writer, request, user, server)
|
||||
return
|
||||
}
|
||||
if len(parts) == 4 && parts[1] == "firewall" && parts[2] == "rules" && request.Method == http.MethodDelete {
|
||||
if !app.authorize(writer, request, user, csrfHash, RoleAdministrator) {
|
||||
return
|
||||
}
|
||||
app.managedDelete(writer, request, user, server, "firewall", parts[3])
|
||||
return
|
||||
}
|
||||
if len(parts) == 2 && parts[1] == "rate-limits" && request.Method == http.MethodPost {
|
||||
if !app.authorize(writer, request, user, csrfHash, RoleAdministrator) {
|
||||
return
|
||||
}
|
||||
app.rateLimitAdd(writer, request, user, server)
|
||||
return
|
||||
}
|
||||
if len(parts) == 3 && parts[1] == "rate-limits" && request.Method == http.MethodDelete {
|
||||
if !app.authorize(writer, request, user, csrfHash, RoleAdministrator) {
|
||||
return
|
||||
}
|
||||
app.managedDelete(writer, request, user, server, "rate-limits", parts[2])
|
||||
return
|
||||
}
|
||||
writeError(writer, 404, "not found", "not_found")
|
||||
@@ -352,18 +413,25 @@ func (app *App) fail2banMutation(writer http.ResponseWriter, request *http.Reque
|
||||
writeError(writer, 502, "agent unavailable", "agent_unavailable")
|
||||
return
|
||||
}
|
||||
if status < 300 {
|
||||
app.store.AddFail2BanEvent(request.Context(), server.ID, input.Jail, input.IP, action, "operator")
|
||||
}
|
||||
writeAgentResponse(writer, status, response)
|
||||
}
|
||||
|
||||
func (app *App) serviceMutation(writer http.ResponseWriter, request *http.Request, user User, server Server, name string) {
|
||||
func (app *App) serviceMutation(writer http.ResponseWriter, request *http.Request, user User, server Server, name, operation string) {
|
||||
if !contains([]string{"start", "stop", "restart", "enable", "disable", "install"}, operation) {
|
||||
writeError(writer, 400, "invalid service operation", "invalid_request")
|
||||
return
|
||||
}
|
||||
status, response, err := app.agent.Do(request.Context(), server, http.MethodPost,
|
||||
"/v1/services/"+url.PathEscape(name)+"/restart", []byte("{}"))
|
||||
"/v1/services/"+url.PathEscape(name)+"/"+operation, []byte("{}"))
|
||||
result := "success"
|
||||
if err != nil || status >= 300 {
|
||||
result = "failure"
|
||||
}
|
||||
app.store.Audit(request.Context(), &user, server.ID, "service.restart", name, nil,
|
||||
map[string]string{"requested": "restart"}, result, app.clientIP(request))
|
||||
app.store.Audit(request.Context(), &user, server.ID, "service."+operation, name, nil,
|
||||
map[string]string{"requested": operation}, result, app.clientIP(request))
|
||||
if err != nil {
|
||||
writeError(writer, 502, "agent unavailable", "agent_unavailable")
|
||||
return
|
||||
@@ -371,6 +439,141 @@ func (app *App) serviceMutation(writer http.ResponseWriter, request *http.Reques
|
||||
writeAgentResponse(writer, status, response)
|
||||
}
|
||||
|
||||
func (app *App) firewallAdd(writer http.ResponseWriter, request *http.Request, user User, server Server) {
|
||||
var input struct {
|
||||
Interface string `json:"interface"`
|
||||
Source string `json:"source"`
|
||||
Protocol string `json:"protocol"`
|
||||
Port int `json:"port"`
|
||||
Target string `json:"target"`
|
||||
Comment string `json:"comment"`
|
||||
}
|
||||
if decodeJSON(writer, request, &input) != nil || !validInterface(input.Interface) || !validIPv4CIDR(input.Source) || !contains([]string{"tcp", "udp"}, input.Protocol) || input.Port < 1 || input.Port > 65535 || !contains([]string{"ACCEPT", "DROP", "REJECT"}, input.Target) || !validComment(input.Comment) {
|
||||
writeError(writer, 400, "invalid firewall rule", "invalid_request")
|
||||
return
|
||||
}
|
||||
app.forwardMutation(writer, request, user, server, "firewall.add", fmt.Sprintf("%s/%s:%d", input.Source, input.Protocol, input.Port), "/v1/firewall/rules", http.MethodPost, input)
|
||||
}
|
||||
|
||||
func (app *App) rateLimitAdd(writer http.ResponseWriter, request *http.Request, user User, server Server) {
|
||||
var input struct {
|
||||
Interface string `json:"interface"`
|
||||
Source string `json:"source"`
|
||||
Protocol string `json:"protocol"`
|
||||
Port int `json:"port"`
|
||||
Rate int `json:"rate_per_second"`
|
||||
Burst int `json:"burst"`
|
||||
Comment string `json:"comment"`
|
||||
}
|
||||
if decodeJSON(writer, request, &input) != nil || !validInterface(input.Interface) || !validIPv4CIDR(input.Source) || !contains([]string{"tcp", "udp"}, input.Protocol) || input.Port < 1 || input.Port > 65535 || input.Rate < 1 || input.Rate > 100000 || input.Burst < 1 || input.Burst > 10000 || !validComment(input.Comment) {
|
||||
writeError(writer, 400, "invalid rate-limit policy", "invalid_request")
|
||||
return
|
||||
}
|
||||
app.forwardMutation(writer, request, user, server, "rate_limit.add", fmt.Sprintf("%s/%s:%d", input.Source, input.Protocol, input.Port), "/v1/rate-limits", http.MethodPost, input)
|
||||
}
|
||||
|
||||
func (app *App) managedDelete(writer http.ResponseWriter, request *http.Request, user User, server Server, kind, id string) {
|
||||
if len(id) != 16 {
|
||||
writeError(writer, 400, "invalid managed object id", "invalid_request")
|
||||
return
|
||||
}
|
||||
for _, character := range id {
|
||||
if !((character >= '0' && character <= '9') || (character >= 'a' && character <= 'f') || (character >= 'A' && character <= 'F')) {
|
||||
writeError(writer, 400, "invalid managed object id", "invalid_request")
|
||||
return
|
||||
}
|
||||
}
|
||||
path := "/v1/firewall/rules/" + id
|
||||
action := "firewall.delete"
|
||||
if kind == "rate-limits" {
|
||||
path = "/v1/rate-limits/" + id
|
||||
action = "rate_limit.delete"
|
||||
}
|
||||
app.forwardMutation(writer, request, user, server, action, id, path, http.MethodDelete, map[string]string{"id": id})
|
||||
}
|
||||
|
||||
func (app *App) fail2banConfigure(writer http.ResponseWriter, request *http.Request, user User, server Server) {
|
||||
var input struct {
|
||||
Jail string `json:"jail"`
|
||||
Enabled bool `json:"enabled"`
|
||||
BanTime int `json:"bantime_seconds"`
|
||||
FindTime int `json:"findtime_seconds"`
|
||||
MaxRetry int `json:"maxretry"`
|
||||
}
|
||||
if decodeJSON(writer, request, &input) != nil || !validSimpleName(input.Jail) || input.BanTime < 10 || input.BanTime > 2592000 || input.FindTime < 10 || input.FindTime > 86400 || input.MaxRetry < 1 || input.MaxRetry > 100 {
|
||||
writeError(writer, 400, "invalid Fail2Ban settings", "invalid_request")
|
||||
return
|
||||
}
|
||||
app.forwardMutation(writer, request, user, server, "fail2ban.configure", input.Jail, "/v1/fail2ban/configure", http.MethodPost, input)
|
||||
}
|
||||
|
||||
func (app *App) forwardMutation(writer http.ResponseWriter, request *http.Request, user User, server Server, action, target, path, method string, input any) {
|
||||
body, _ := json.Marshal(input)
|
||||
status, response, err := app.agent.Do(request.Context(), server, method, path, body)
|
||||
if err == nil && status < 300 && (strings.HasPrefix(action, "firewall.") || strings.HasPrefix(action, "rate_limit.")) {
|
||||
var pending struct {
|
||||
TransactionID string `json:"transaction_id"`
|
||||
}
|
||||
if json.Unmarshal(response, &pending) != nil || len(pending.TransactionID) != 16 {
|
||||
err = errors.New("agent returned invalid firewall transaction")
|
||||
} else {
|
||||
confirmPath := "/v1/firewall/transactions/" + pending.TransactionID + "/confirm"
|
||||
confirmStatus, confirmResponse, confirmErr := app.agent.Do(request.Context(), server, http.MethodPost, confirmPath, []byte("{}"))
|
||||
if confirmErr != nil || confirmStatus >= 300 {
|
||||
err = errors.New("firewall confirmation failed; safety rollback remains armed")
|
||||
} else {
|
||||
status = confirmStatus
|
||||
response = confirmResponse
|
||||
}
|
||||
}
|
||||
}
|
||||
result := "success"
|
||||
if err != nil || status >= 300 {
|
||||
result = "failure"
|
||||
}
|
||||
app.store.Audit(request.Context(), &user, server.ID, action, target, nil, input, result, app.clientIP(request))
|
||||
if err != nil {
|
||||
writeError(writer, 502, "agent unavailable", "agent_unavailable")
|
||||
return
|
||||
}
|
||||
writeAgentResponse(writer, status, response)
|
||||
}
|
||||
|
||||
func contains(values []string, wanted string) bool {
|
||||
for _, value := range values {
|
||||
if value == wanted {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
func validInterface(value string) bool {
|
||||
if len(value) < 1 || len(value) > 15 {
|
||||
return false
|
||||
}
|
||||
for _, c := range value {
|
||||
if !(c >= 'a' && c <= 'z') && !(c >= 'A' && c <= 'Z') && !(c >= '0' && c <= '9') && !strings.ContainsRune("_.-", c) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
func validIPv4CIDR(value string) bool {
|
||||
ip, _, err := net.ParseCIDR(value)
|
||||
return err == nil && ip.To4() != nil
|
||||
}
|
||||
func validComment(value string) bool {
|
||||
if len(value) > 80 {
|
||||
return false
|
||||
}
|
||||
for _, c := range value {
|
||||
if !(c >= 'a' && c <= 'z') && !(c >= 'A' && c <= 'Z') && !(c >= '0' && c <= '9') && !strings.ContainsRune(" ._-", c) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (app *App) clientIP(request *http.Request) string {
|
||||
if app.config.TrustProxy {
|
||||
if value := request.Header.Get("X-Real-IP"); net.ParseIP(value) != nil {
|
||||
@@ -434,6 +637,17 @@ func queryLimit(request *http.Request, fallback, maximum int) int {
|
||||
return value
|
||||
}
|
||||
|
||||
func queryHours(request *http.Request, fallback, maximum int) int {
|
||||
value, err := strconv.Atoi(request.URL.Query().Get("hours"))
|
||||
if err != nil || value < 1 {
|
||||
return fallback
|
||||
}
|
||||
if value > maximum {
|
||||
return maximum
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func writeAgentResponse(writer http.ResponseWriter, status int, payload []byte) {
|
||||
if status < 100 || status > 599 {
|
||||
status = http.StatusBadGateway
|
||||
|
||||
@@ -54,6 +54,15 @@ func TestValidators(t *testing.T) {
|
||||
if !validServiceName("wg-quick@wg0") || validServiceName("nginx/restart") {
|
||||
t.Fatal("service validation failed")
|
||||
}
|
||||
if !validInterface("wg0") || validInterface("../../eth0") {
|
||||
t.Fatal("interface validation failed")
|
||||
}
|
||||
if !validIPv4CIDR("192.168.50.0/24") || validIPv4CIDR("2001:db8::/64") || validIPv4CIDR("0.0.0.0/99") {
|
||||
t.Fatal("firewall CIDR validation failed")
|
||||
}
|
||||
if !validComment("web access") || validComment("bad;reboot") {
|
||||
t.Fatal("firewall comment validation failed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPercent(t *testing.T) {
|
||||
|
||||
@@ -75,6 +75,35 @@ CREATE TABLE IF NOT EXISTS agent_log_entries (
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS agent_log_entries_time_idx ON agent_log_entries(event_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS agent_log_entries_server_time_idx ON agent_log_entries(server_id, event_at DESC);
|
||||
CREATE TABLE IF NOT EXISTS connection_samples (
|
||||
server_id text NOT NULL REFERENCES servers(id) ON DELETE CASCADE,
|
||||
sampled_at timestamptz NOT NULL DEFAULT now(),
|
||||
protocol text NOT NULL,
|
||||
local_address text NOT NULL,
|
||||
local_port integer NOT NULL CHECK (local_port BETWEEN 0 AND 65535),
|
||||
remote_address text NOT NULL,
|
||||
remote_port integer NOT NULL CHECK (remote_port BETWEEN 0 AND 65535),
|
||||
state text NOT NULL,
|
||||
process text NOT NULL DEFAULT '',
|
||||
connection_count integer NOT NULL CHECK (connection_count > 0)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS connection_samples_server_time_idx ON connection_samples(server_id, sampled_at DESC);
|
||||
CREATE TABLE IF NOT EXISTS fail2ban_events (
|
||||
id bigserial PRIMARY KEY,
|
||||
server_id text NOT NULL REFERENCES servers(id) ON DELETE CASCADE,
|
||||
event_at timestamptz NOT NULL DEFAULT now(),
|
||||
jail text NOT NULL,
|
||||
ip text NOT NULL,
|
||||
action text NOT NULL CHECK (action IN ('ban','unban')),
|
||||
source text NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS fail2ban_events_server_time_idx ON fail2ban_events(server_id, event_at DESC);
|
||||
CREATE TABLE IF NOT EXISTS fail2ban_ban_state (
|
||||
server_id text NOT NULL REFERENCES servers(id) ON DELETE CASCADE,
|
||||
jail text NOT NULL,
|
||||
ip text NOT NULL,
|
||||
PRIMARY KEY(server_id,jail,ip)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS audit_events (
|
||||
id bigserial PRIMARY KEY,
|
||||
user_id text REFERENCES users(id) ON DELETE SET NULL,
|
||||
@@ -139,6 +168,37 @@ type AgentLogEntry struct {
|
||||
PID *int `json:"pid,omitempty"`
|
||||
}
|
||||
|
||||
type ConnectionHistoryRow struct {
|
||||
Protocol string `json:"protocol"`
|
||||
LocalAddress string `json:"local_address"`
|
||||
LocalPort int `json:"local_port"`
|
||||
RemoteAddress string `json:"remote_address"`
|
||||
Process string `json:"process"`
|
||||
Samples int64 `json:"samples"`
|
||||
Peak int `json:"peak"`
|
||||
FirstSeen time.Time `json:"first_seen"`
|
||||
LastSeen time.Time `json:"last_seen"`
|
||||
}
|
||||
|
||||
type ConnectionTimelinePoint struct {
|
||||
SampledAt time.Time `json:"sampled_at"`
|
||||
Count int64 `json:"count"`
|
||||
}
|
||||
|
||||
type ConnectionHistory struct {
|
||||
Rows []ConnectionHistoryRow `json:"rows"`
|
||||
Timeline []ConnectionTimelinePoint `json:"timeline"`
|
||||
}
|
||||
|
||||
type Fail2BanEvent struct {
|
||||
ID int64 `json:"id"`
|
||||
EventAt time.Time `json:"event_at"`
|
||||
Jail string `json:"jail"`
|
||||
IP string `json:"ip"`
|
||||
Action string `json:"action"`
|
||||
Source string `json:"source"`
|
||||
}
|
||||
|
||||
type MetricPoint struct {
|
||||
ServerID string
|
||||
ServerName string
|
||||
@@ -372,6 +432,84 @@ func (store *Store) Logs(ctx context.Context, serverID string, limit int) ([]Age
|
||||
return entries, rows.Err()
|
||||
}
|
||||
|
||||
func (store *Store) ConnectionHistory(ctx context.Context, serverID string, since time.Time) (ConnectionHistory, error) {
|
||||
result := ConnectionHistory{Rows: []ConnectionHistoryRow{}, Timeline: []ConnectionTimelinePoint{}}
|
||||
rows, err := store.pool.Query(ctx, `SELECT protocol,local_address,local_port,remote_address,process,
|
||||
SUM(connection_count),MAX(connection_count),MIN(sampled_at),MAX(sampled_at)
|
||||
FROM connection_samples WHERE server_id=$1 AND sampled_at >= $2
|
||||
GROUP BY protocol,local_address,local_port,remote_address,process
|
||||
ORDER BY SUM(connection_count) DESC LIMIT 500`, serverID, since)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var row ConnectionHistoryRow
|
||||
if err = rows.Scan(&row.Protocol, &row.LocalAddress, &row.LocalPort, &row.RemoteAddress, &row.Process, &row.Samples, &row.Peak, &row.FirstSeen, &row.LastSeen); err != nil {
|
||||
return result, err
|
||||
}
|
||||
result.Rows = append(result.Rows, row)
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
return result, err
|
||||
}
|
||||
timeline, err := store.pool.Query(ctx, `SELECT date_bin(interval '5 minutes',sampled_at,timestamptz 'epoch'),SUM(connection_count)
|
||||
FROM connection_samples WHERE server_id=$1 AND sampled_at >= $2
|
||||
GROUP BY 1 ORDER BY 1 LIMIT 9000`, serverID, since)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
defer timeline.Close()
|
||||
for timeline.Next() {
|
||||
var point ConnectionTimelinePoint
|
||||
if err = timeline.Scan(&point.SampledAt, &point.Count); err != nil {
|
||||
return result, err
|
||||
}
|
||||
result.Timeline = append(result.Timeline, point)
|
||||
}
|
||||
return result, timeline.Err()
|
||||
}
|
||||
|
||||
func (store *Store) Fail2BanEvents(ctx context.Context, serverID string, limit int) ([]Fail2BanEvent, error) {
|
||||
rows, err := store.pool.Query(ctx, `SELECT id,event_at,jail,ip,action,source FROM fail2ban_events
|
||||
WHERE server_id=$1 ORDER BY event_at DESC LIMIT $2`, serverID, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
events := []Fail2BanEvent{}
|
||||
for rows.Next() {
|
||||
var event Fail2BanEvent
|
||||
if err = rows.Scan(&event.ID, &event.EventAt, &event.Jail, &event.IP, &event.Action, &event.Source); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
events = append(events, event)
|
||||
}
|
||||
return events, rows.Err()
|
||||
}
|
||||
|
||||
func (store *Store) AddFail2BanEvent(ctx context.Context, serverID, jail, ip, action, source string) {
|
||||
tx, err := store.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
log.Printf("write Fail2Ban event: %v", err)
|
||||
return
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
if _, err = tx.Exec(ctx, `INSERT INTO fail2ban_events(server_id,jail,ip,action,source) VALUES($1,$2,$3,$4,$5)`, serverID, jail, ip, action, source); err == nil {
|
||||
if action == "ban" {
|
||||
_, err = tx.Exec(ctx, `INSERT INTO fail2ban_ban_state(server_id,jail,ip) VALUES($1,$2,$3) ON CONFLICT DO NOTHING`, serverID, jail, ip)
|
||||
} else {
|
||||
_, err = tx.Exec(ctx, `DELETE FROM fail2ban_ban_state WHERE server_id=$1 AND jail=$2 AND ip=$3`, serverID, jail, ip)
|
||||
}
|
||||
}
|
||||
if err == nil {
|
||||
err = tx.Commit(ctx)
|
||||
}
|
||||
if err != nil {
|
||||
log.Printf("write Fail2Ban event: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (store *Store) MetricSeries(ctx context.Context) ([]map[string]string, error) {
|
||||
rows, err := store.pool.Query(ctx, "SELECT id,name FROM servers ORDER BY id LIMIT 1000")
|
||||
if err != nil {
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
@@ -41,6 +42,8 @@ func (poller *Poller) Run(ctx context.Context) {
|
||||
if _, err := poller.store.pool.Exec(ctx, "DELETE FROM agent_log_entries WHERE event_at < $1", time.Now().Add(-poller.logRetention)); err != nil {
|
||||
log.Printf("log retention cleanup: %v", err)
|
||||
}
|
||||
_, _ = poller.store.pool.Exec(ctx, "DELETE FROM connection_samples WHERE sampled_at < $1", time.Now().Add(-poller.metricRetention))
|
||||
_, _ = poller.store.pool.Exec(ctx, "DELETE FROM fail2ban_events WHERE event_at < $1", time.Now().Add(-poller.logRetention))
|
||||
_, _ = poller.store.pool.Exec(ctx, "DELETE FROM sessions WHERE expires_at < now()")
|
||||
}
|
||||
}
|
||||
@@ -119,13 +122,26 @@ func (poller *Poller) pollServer(ctx context.Context, server Server) {
|
||||
}
|
||||
}
|
||||
}
|
||||
type socket struct {
|
||||
Protocol string `json:"protocol"`
|
||||
LocalAddress string `json:"local_address"`
|
||||
LocalPort int `json:"local_port"`
|
||||
RemoteAddress string `json:"remote_address"`
|
||||
RemotePort int `json:"remote_port"`
|
||||
State string `json:"state"`
|
||||
Process *string `json:"process"`
|
||||
}
|
||||
connections := 0
|
||||
connectionRows := []socket{}
|
||||
if code, payload, callErr := poller.agent.Do(ctx, server, http.MethodGet, "/v1/network/connections", nil); callErr == nil && code == 200 {
|
||||
var items []json.RawMessage
|
||||
if json.Unmarshal(payload, &items) == nil {
|
||||
connections = len(items)
|
||||
if json.Unmarshal(payload, &connectionRows) == nil {
|
||||
if len(connectionRows) > 10000 {
|
||||
connectionRows = connectionRows[:10000]
|
||||
}
|
||||
connections = len(connectionRows)
|
||||
}
|
||||
}
|
||||
bans := poller.pollFail2Ban(ctx, server)
|
||||
txDB, err := poller.store.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return
|
||||
@@ -134,7 +150,41 @@ func (poller *Poller) pollServer(ctx context.Context, server Server) {
|
||||
_, err = txDB.Exec(ctx, "UPDATE servers SET status='online',hostname=$2,last_seen=now(),updated_at=now() WHERE id=$1", server.ID, status.Hostname)
|
||||
if err == nil {
|
||||
_, err = txDB.Exec(ctx, `INSERT INTO metric_samples(server_id,cpu,memory,disk,rx,tx,connections,bans)
|
||||
VALUES($1,$2,$3,$4,$5,$6,$7,0)`, server.ID, status.CPU, memory, disk, rx, tx, connections)
|
||||
VALUES($1,$2,$3,$4,$5,$6,$7,$8)`, server.ID, status.CPU, memory, disk, rx, tx, connections, bans)
|
||||
}
|
||||
if err == nil {
|
||||
type aggregate struct {
|
||||
row socket
|
||||
count int
|
||||
}
|
||||
grouped := map[string]aggregate{}
|
||||
for _, row := range connectionRows {
|
||||
if net.ParseIP(row.LocalAddress) == nil || net.ParseIP(row.RemoteAddress) == nil || row.LocalPort < 0 || row.LocalPort > 65535 || row.RemotePort < 0 || row.RemotePort > 65535 || len(row.Protocol) > 8 || len(row.State) > 32 {
|
||||
continue
|
||||
}
|
||||
process := ""
|
||||
if row.Process != nil {
|
||||
process = truncateUTF8(*row.Process, 128)
|
||||
}
|
||||
key := fmt.Sprintf("%s\x00%s\x00%d\x00%s\x00%d\x00%s\x00%s", row.Protocol, row.LocalAddress, row.LocalPort, row.RemoteAddress, row.RemotePort, row.State, process)
|
||||
entry := grouped[key]
|
||||
entry.row = row
|
||||
entry.row.Process = &process
|
||||
entry.count++
|
||||
grouped[key] = entry
|
||||
}
|
||||
inserted := 0
|
||||
for _, entry := range grouped {
|
||||
if inserted >= 2000 {
|
||||
break
|
||||
}
|
||||
_, err = txDB.Exec(ctx, `INSERT INTO connection_samples(server_id,protocol,local_address,local_port,remote_address,remote_port,state,process,connection_count)
|
||||
VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9)`, server.ID, entry.row.Protocol, entry.row.LocalAddress, entry.row.LocalPort, entry.row.RemoteAddress, entry.row.RemotePort, entry.row.State, *entry.row.Process, entry.count)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
inserted++
|
||||
}
|
||||
}
|
||||
if err == nil {
|
||||
err = txDB.Commit(ctx)
|
||||
@@ -145,6 +195,78 @@ func (poller *Poller) pollServer(ctx context.Context, server Server) {
|
||||
poller.pollLogs(ctx, server)
|
||||
}
|
||||
|
||||
func (poller *Poller) pollFail2Ban(ctx context.Context, server Server) int {
|
||||
status, payload, err := poller.agent.Do(ctx, server, http.MethodGet, "/v1/fail2ban/status", nil)
|
||||
if err != nil || status != http.StatusOK {
|
||||
return 0
|
||||
}
|
||||
var response struct {
|
||||
JailStatuses []struct {
|
||||
Jail string `json:"jail"`
|
||||
Banned []string `json:"banned_ips"`
|
||||
} `json:"jail_statuses"`
|
||||
}
|
||||
if json.Unmarshal(payload, &response) != nil || len(response.JailStatuses) > 64 {
|
||||
return 0
|
||||
}
|
||||
current := map[string][2]string{}
|
||||
total := 0
|
||||
for _, jail := range response.JailStatuses {
|
||||
if !validSimpleName(jail.Jail) || len(jail.Banned) > 10000 {
|
||||
continue
|
||||
}
|
||||
for _, ip := range jail.Banned {
|
||||
if net.ParseIP(ip) != nil {
|
||||
current[jail.Jail+"\x00"+ip] = [2]string{jail.Jail, ip}
|
||||
total++
|
||||
}
|
||||
}
|
||||
}
|
||||
rows, err := poller.store.pool.Query(ctx, "SELECT jail,ip FROM fail2ban_ban_state WHERE server_id=$1", server.ID)
|
||||
if err != nil {
|
||||
return total
|
||||
}
|
||||
previous := map[string][2]string{}
|
||||
for rows.Next() {
|
||||
var jail, ip string
|
||||
if rows.Scan(&jail, &ip) == nil {
|
||||
previous[jail+"\x00"+ip] = [2]string{jail, ip}
|
||||
}
|
||||
}
|
||||
rows.Close()
|
||||
tx, err := poller.store.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return total
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
for key, value := range current {
|
||||
if _, found := previous[key]; !found {
|
||||
_, err = tx.Exec(ctx, "INSERT INTO fail2ban_events(server_id,jail,ip,action,source) VALUES($1,$2,$3,'ban','agent')", server.ID, value[0], value[1])
|
||||
if err != nil {
|
||||
return total
|
||||
}
|
||||
}
|
||||
}
|
||||
for key, value := range previous {
|
||||
if _, found := current[key]; !found {
|
||||
_, err = tx.Exec(ctx, "INSERT INTO fail2ban_events(server_id,jail,ip,action,source) VALUES($1,$2,$3,'unban','agent')", server.ID, value[0], value[1])
|
||||
if err != nil {
|
||||
return total
|
||||
}
|
||||
}
|
||||
}
|
||||
if _, err = tx.Exec(ctx, "DELETE FROM fail2ban_ban_state WHERE server_id=$1", server.ID); err != nil {
|
||||
return total
|
||||
}
|
||||
for _, value := range current {
|
||||
if _, err = tx.Exec(ctx, "INSERT INTO fail2ban_ban_state(server_id,jail,ip) VALUES($1,$2,$3)", server.ID, value[0], value[1]); err != nil {
|
||||
return total
|
||||
}
|
||||
}
|
||||
_ = tx.Commit(ctx)
|
||||
return total
|
||||
}
|
||||
|
||||
func (poller *Poller) pollLogs(ctx context.Context, server Server) {
|
||||
var cursor string
|
||||
var since time.Time
|
||||
|
||||
Reference in New Issue
Block a user