feat: add Bastillion-style SSH key enforcement worker

This commit is contained in:
2026-04-06 00:17:03 +02:00
parent 3a843354b6
commit 8b9de9e83d
11 changed files with 992 additions and 3 deletions

View File

@@ -25,6 +25,7 @@
- **Temporary Access** — Schedule time-limited access with automatic expiry (key removal, user disable, or user deletion) - **Temporary Access** — Schedule time-limited access with automatic expiry (key removal, user disable, or user deletion)
- **Three-Tier Roles** — Owner, Admin, and User with distinct permissions - **Three-Tier Roles** — Owner, Admin, and User with distinct permissions
- **User Invitations** — Invite users via secure email links - **User Invitations** — Invite users via secure email links
- **Key Enforcement** — Bastillion-style enforced key management: automatically detect and remove unauthorized SSH keys from servers
- **Two-Factor Authentication** — TOTP-based MFA, optionally enforced for all users - **Two-Factor Authentication** — TOTP-based MFA, optionally enforced for all users
- **Password Policies & Account Lockout** — Configurable complexity rules and brute-force protection - **Password Policies & Account Lockout** — Configurable complexity rules and brute-force protection
- **Audit Log** — Every action tracked with user, IP, timestamp, and details - **Audit Log** — Every action tracked with user, IP, timestamp, and details

View File

@@ -24,6 +24,7 @@ import (
"git.techniverse.net/scriptos/keywarden/internal/mail" "git.techniverse.net/scriptos/keywarden/internal/mail"
"git.techniverse.net/scriptos/keywarden/internal/security" "git.techniverse.net/scriptos/keywarden/internal/security"
"git.techniverse.net/scriptos/keywarden/internal/servers" "git.techniverse.net/scriptos/keywarden/internal/servers"
"git.techniverse.net/scriptos/keywarden/internal/worker"
"git.techniverse.net/scriptos/keywarden/web" "git.techniverse.net/scriptos/keywarden/web"
) )
@@ -76,6 +77,7 @@ func main() {
deploySvc := deploy.NewService(db) deploySvc := deploy.NewService(db)
auditSvc := audit.NewService(db) auditSvc := audit.NewService(db)
cronSvc := cron.NewService(db, deploySvc, keysSvc, serversSvc, auditSvc) cronSvc := cron.NewService(db, deploySvc, keysSvc, serversSvc, auditSvc)
workerSvc := worker.NewService(db, deploySvc, keysSvc, serversSvc, auditSvc)
mailSvc := mail.NewService(cfg) mailSvc := mail.NewService(cfg)
// Create default owner if no users exist (password is auto-generated) // Create default owner if no users exist (password is auto-generated)
@@ -116,7 +118,7 @@ func main() {
} }
// Setup HTTP handlers // Setup HTTP handlers
handler := handlers.New(authSvc, keysSvc, serversSvc, deploySvc, auditSvc, cronSvc, mailSvc, db, web.TemplateFS, web.StaticFS, cfg.DataDir, cfg.SecureCookies, cfg.BaseURL) handler := handlers.New(authSvc, keysSvc, serversSvc, deploySvc, auditSvc, cronSvc, workerSvc, mailSvc, db, web.TemplateFS, web.StaticFS, cfg.DataDir, cfg.SecureCookies, cfg.BaseURL)
mux := http.NewServeMux() mux := http.NewServeMux()
handler.RegisterRoutes(mux) handler.RegisterRoutes(mux)
@@ -142,6 +144,10 @@ func main() {
cronSvc.Start() cronSvc.Start()
defer cronSvc.Stop() defer cronSvc.Stop()
// Start key enforcement worker
workerSvc.Start()
defer workerSvc.Stop()
// Start server // Start server
addr := ":" + cfg.Port addr := ":" + cfg.Port
logging.Info("Server starting on http://0.0.0.0%s", addr) logging.Info("Server starting on http://0.0.0.0%s", addr)

View File

@@ -34,6 +34,7 @@ Keywarden provides a clean web UI to generate, import, and securely store SSH ke
- **Temporary Access (Cron Jobs)** — Schedule time-limited access with automatic key removal, user disabling, or user deletion on expiry - **Temporary Access (Cron Jobs)** — Schedule time-limited access with automatic key removal, user disabling, or user deletion on expiry
- **Three-Tier Role System** — Owner, Admin, and User roles with clear permission boundaries - **Three-Tier Role System** — Owner, Admin, and User roles with clear permission boundaries
- **User Invitations** — Invite new users via secure email links with self-service password setup - **User Invitations** — Invite new users via secure email links with self-service password setup
- **Key Enforcement** — Bastillion-style enforced key management: detect and remove unauthorized SSH keys automatically
- **TOTP Two-Factor Authentication** — Optional or enforced MFA for all users - **TOTP Two-Factor Authentication** — Optional or enforced MFA for all users
- **Password Policies** — Configurable complexity requirements with account lockout - **Password Policies** — Configurable complexity requirements with account lockout
- **Email Notifications** — Login alerts and invitation emails via SMTP - **Email Notifications** — Login alerts and invitation emails via SMTP

View File

@@ -234,6 +234,14 @@ Navigate to **Admin Settings** (owner only) to configure:
- **Account Lockout** — Number of failed attempts before lockout and lockout duration - **Account Lockout** — Number of failed attempts before lockout and lockout duration
- **MFA Enforcement** — Require all users to enable TOTP MFA - **MFA Enforcement** — Require all users to enable TOTP MFA
### Key Enforcement
- **Enforcement Mode** — Disabled (default), Monitor (log only), or Enforce (auto-remove unauthorized keys)
- **Check Interval** — How often the worker scans servers (11440 minutes, default: 15)
- **Run Now** — Trigger an immediate enforcement check
See [Security — Key Enforcement](security.md#key-enforcement-bastillion-style) for details.
### Master Key ### Master Key
- View the system master key's public key and fingerprint - View the system master key's public key and fingerprint

View File

@@ -39,6 +39,7 @@ internal/
security/ ← CSRF, security headers, rate limiting, proxy detection security/ ← CSRF, security headers, rate limiting, proxy detection
servers/ ← Server and server group management, access assignments servers/ ← Server and server group management, access assignments
sshutil/ ← SSH key generation (RSA, Ed25519, Ed448) sshutil/ ← SSH key generation (RSA, Ed25519, Ed448)
worker/ ← Background key enforcement worker (Bastillion-style)
web/ web/
embed.go ← Go embed directives for templates and static files embed.go ← Go embed directives for templates and static files
static/ ← CSS, JS, fonts (Tabler UI framework) static/ ← CSS, JS, fonts (Tabler UI framework)
@@ -59,7 +60,8 @@ web/
10. **Start session cleanup** goroutine (removes expired sessions every minute) 10. **Start session cleanup** goroutine (removes expired sessions every minute)
11. **Apply middleware chain**: request logger → security headers → rate limiting → size limiting → CSRF 11. **Apply middleware chain**: request logger → security headers → rate limiting → size limiting → CSRF
12. **Start cron scheduler** (checks for pending jobs every 30 seconds) 12. **Start cron scheduler** (checks for pending jobs every 30 seconds)
13. **Start HTTP server** 13. **Start key enforcement worker** (if enabled in Admin Settings)
14. **Start HTTP server**
## Database Design ## Database Design

View File

@@ -209,3 +209,58 @@ When deploying keys to servers, Keywarden:
8. **Network isolation**: Restrict access to Keywarden and managed servers to trusted networks 8. **Network isolation**: Restrict access to Keywarden and managed servers to trusted networks
9. **Keep the encryption key safe**: Back up `KEYWARDEN_ENCRYPTION_KEY` securely — losing it means losing all private keys 9. **Keep the encryption key safe**: Back up `KEYWARDEN_ENCRYPTION_KEY` securely — losing it means losing all private keys
10. **Monitor the audit log**: Review login activity and deployment actions regularly 10. **Monitor the audit log**: Review login activity and deployment actions regularly
11. **Enable key enforcement**: Use enforce mode to ensure only Keywarden-managed keys exist on your servers
## Key Enforcement (Bastillion-Style)
Keywarden includes an enforced key management feature inspired by [Bastillion](https://www.bastillion.io/). When enabled, a background worker periodically connects to all managed servers and ensures that only authorized SSH keys are present in `authorized_keys` files.
### How It Works
1. The enforcement worker runs at a configurable interval (default: 15 minutes)
2. For each managed server and system user, it reads the current `authorized_keys`
3. It compares the keys against the **desired state** derived from:
- All active access assignments (desired_state = "present")
- All active cron jobs (temporary access that has not yet expired)
- All direct key deployments (via the Deploy page)
- The system master key (always authorized)
4. Unauthorized keys (not managed by Keywarden) are detected
5. Depending on the mode, unauthorized keys are either logged or removed
### Modes
| Mode | Behavior |
|---|---|
| **Disabled** | No enforcement checks (default) |
| **Monitor** | Detects unauthorized keys and logs them in the audit log, but does not remove them |
| **Enforce** | Detects unauthorized keys and **removes them automatically**, replacing `authorized_keys` with only the authorized set |
### Configuration
Key enforcement is configured in **Admin Settings → Key Enforcement**:
- **Enforcement Mode**: Disabled / Monitor / Enforce
- **Check Interval**: How often the worker checks servers (11440 minutes)
- **Run Now**: Trigger an immediate enforcement check
### Audit Trail
All enforcement actions are recorded in the audit log:
| Action | Description |
|---|---|
| `enforcement_run` | An enforcement cycle completed (with summary) |
| `enforcement_drift` | Unauthorized keys detected on a server |
| `enforcement_applied` | Unauthorized keys were removed from a server |
| `enforcement_failed` | An enforcement action failed (connection error, etc.) |
| `enforcement_settings_changed` | Enforcement settings were modified |
### Important Notes
- The system master key is **always** considered authorized and will never be removed
- Enforcement covers all system users that have active access assignments, cron jobs, or direct deployments in Keywarden
- The server's admin user (used for SSH connections) is always checked
- Enforcement requires the system master key to be deployed on target servers
- In **enforce** mode, `authorized_keys` is atomically replaced (write to temp file, then move)
- Manual runs can be triggered from the Admin Settings page

View File

@@ -107,6 +107,13 @@ const (
ActionInvitationSendFailed = "invitation_send_failed" ActionInvitationSendFailed = "invitation_send_failed"
ActionInvitationAccepted = "invitation_accepted" ActionInvitationAccepted = "invitation_accepted"
ActionInvitationFailed = "invitation_failed" ActionInvitationFailed = "invitation_failed"
// Key Enforcement
ActionEnforcementRun = "enforcement_run"
ActionEnforcementDrift = "enforcement_drift"
ActionEnforcementApplied = "enforcement_applied"
ActionEnforcementFailed = "enforcement_failed"
ActionEnforcementSettings = "enforcement_settings_changed"
) )
// AuditEntry extends AuditLog with the username for display // AuditEntry extends AuditLog with the username for display

View File

@@ -655,3 +655,110 @@ func (s *Service) GetDeployments(userID int64) ([]map[string]interface{}, error)
} }
return deployments, nil return deployments, nil
} }
// ReadAuthorizedKeys reads the current authorized_keys for a system user on a remote server.
// Returns the list of key lines (non-empty, non-comment lines).
func (s *Service) ReadAuthorizedKeys(server *models.Server, authPrivateKey []byte, systemUser string) ([]string, error) {
signer, err := ssh.ParsePrivateKey(authPrivateKey)
if err != nil {
return nil, fmt.Errorf("failed to parse authentication key: %w", err)
}
config := &ssh.ClientConfig{
User: server.Username,
Auth: []ssh.AuthMethod{
ssh.PublicKeys(signer),
},
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
Timeout: 10 * time.Second,
}
addr := net.JoinHostPort(server.Hostname, fmt.Sprintf("%d", server.Port))
client, err := ssh.Dial("tcp", addr, config)
if err != nil {
return nil, fmt.Errorf("failed to connect to server: %w", err)
}
defer client.Close()
session, err := client.NewSession()
if err != nil {
return nil, fmt.Errorf("failed to create session: %w", err)
}
defer session.Close()
homeDir := fmt.Sprintf("/home/%s", systemUser)
if systemUser == "root" {
homeDir = "/root"
}
cmd := fmt.Sprintf(`cat %s/.ssh/authorized_keys 2>/dev/null || echo ""`, homeDir)
output, err := session.Output(cmd)
if err != nil {
return nil, fmt.Errorf("failed to read authorized_keys: %w", err)
}
var keys []string
for _, line := range strings.Split(string(output), "\n") {
line = strings.TrimSpace(line)
if line != "" && !strings.HasPrefix(line, "#") {
keys = append(keys, line)
}
}
return keys, nil
}
// WriteAuthorizedKeys replaces the entire authorized_keys file for a system user on a remote server
// with the provided set of keys. This is the enforcement function.
func (s *Service) WriteAuthorizedKeys(server *models.Server, authPrivateKey []byte, systemUser string, authorizedKeys []string) error {
signer, err := ssh.ParsePrivateKey(authPrivateKey)
if err != nil {
return fmt.Errorf("failed to parse authentication key: %w", err)
}
config := &ssh.ClientConfig{
User: server.Username,
Auth: []ssh.AuthMethod{
ssh.PublicKeys(signer),
},
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
Timeout: 10 * time.Second,
}
addr := net.JoinHostPort(server.Hostname, fmt.Sprintf("%d", server.Port))
client, err := ssh.Dial("tcp", addr, config)
if err != nil {
return fmt.Errorf("failed to connect to server: %w", err)
}
defer client.Close()
session, err := client.NewSession()
if err != nil {
return fmt.Errorf("failed to create session: %w", err)
}
defer session.Close()
homeDir := fmt.Sprintf("/home/%s", systemUser)
if systemUser == "root" {
homeDir = "/root"
}
// Build the authorized_keys content
content := strings.Join(authorizedKeys, "\n")
if !strings.HasSuffix(content, "\n") {
content += "\n"
}
// Use printf to write the content to avoid shell interpretation issues
// First write to a temp file, then atomically move it
escapedContent := strings.ReplaceAll(content, "'", "'\\''")
cmd := fmt.Sprintf(
`mkdir -p %s/.ssh && chmod 700 %s/.ssh && printf '%%s' '%s' > %s/.ssh/authorized_keys.tmp && mv %s/.ssh/authorized_keys.tmp %s/.ssh/authorized_keys && chmod 600 %s/.ssh/authorized_keys && chown '%s':'%s' %s/.ssh/authorized_keys`,
homeDir, homeDir, escapedContent, homeDir, homeDir, homeDir, homeDir, systemUser, systemUser, homeDir,
)
if err := session.Run(cmd); err != nil {
return fmt.Errorf("failed to write authorized_keys: %w", err)
}
return nil
}

View File

@@ -39,6 +39,7 @@ import (
"git.techniverse.net/scriptos/keywarden/internal/models" "git.techniverse.net/scriptos/keywarden/internal/models"
"git.techniverse.net/scriptos/keywarden/internal/security" "git.techniverse.net/scriptos/keywarden/internal/security"
"git.techniverse.net/scriptos/keywarden/internal/servers" "git.techniverse.net/scriptos/keywarden/internal/servers"
"git.techniverse.net/scriptos/keywarden/internal/worker"
) )
// sessionData holds session metadata for timeout tracking // sessionData holds session metadata for timeout tracking
@@ -56,6 +57,7 @@ type Handler struct {
deploy *deploy.Service deploy *deploy.Service
audit *audit.Service audit *audit.Service
cron *cron.Service cron *cron.Service
worker *worker.Service
mail *mail.Service mail *mail.Service
db *database.DB // direct database access for backup/restore db *database.DB // direct database access for backup/restore
templates map[string]*template.Template templates map[string]*template.Template
@@ -169,6 +171,9 @@ type PageData struct {
// System Information // System Information
SystemInfo *SystemInfo SystemInfo *SystemInfo
// Key Enforcement
EnforcementStatus map[string]string
} }
// SystemInfo holds runtime system information for the settings page // SystemInfo holds runtime system information for the settings page
@@ -242,7 +247,7 @@ func formatUptime(start time.Time) string {
} }
// New creates a new Handler // New creates a new Handler
func New(authSvc *auth.Service, keysSvc *keys.Service, serversSvc *servers.Service, deploySvc *deploy.Service, auditSvc *audit.Service, cronSvc *cron.Service, mailSvc *mail.Service, db *database.DB, templateFS embed.FS, staticFS embed.FS, dataDir string, secureCookies bool, baseURL string) *Handler { func New(authSvc *auth.Service, keysSvc *keys.Service, serversSvc *servers.Service, deploySvc *deploy.Service, auditSvc *audit.Service, cronSvc *cron.Service, workerSvc *worker.Service, mailSvc *mail.Service, db *database.DB, templateFS embed.FS, staticFS embed.FS, dataDir string, secureCookies bool, baseURL string) *Handler {
// Create sub-FS so /static/css/... maps to static/css/... in embed // Create sub-FS so /static/css/... maps to static/css/... in embed
staticSub, err := fs.Sub(staticFS, "static") staticSub, err := fs.Sub(staticFS, "static")
if err != nil { if err != nil {
@@ -262,6 +267,7 @@ func New(authSvc *auth.Service, keysSvc *keys.Service, serversSvc *servers.Servi
deploy: deploySvc, deploy: deploySvc,
audit: auditSvc, audit: auditSvc,
cron: cronSvc, cron: cronSvc,
worker: workerSvc,
mail: mailSvc, mail: mailSvc,
db: db, db: db,
sessions: make(map[string]*sessionData), sessions: make(map[string]*sessionData),
@@ -432,6 +438,7 @@ func (h *Handler) RegisterRoutes(mux *http.ServeMux) {
mux.HandleFunc("/admin/masterkey/regenerate", h.requireOwner(h.handleMasterKeyRegenerate)) mux.HandleFunc("/admin/masterkey/regenerate", h.requireOwner(h.handleMasterKeyRegenerate))
mux.HandleFunc("/admin/backup/export", h.requireOwner(h.handleBackupExport)) mux.HandleFunc("/admin/backup/export", h.requireOwner(h.handleBackupExport))
mux.HandleFunc("/admin/backup/import", h.requireOwner(h.handleBackupImport)) mux.HandleFunc("/admin/backup/import", h.requireOwner(h.handleBackupImport))
mux.HandleFunc("/admin/enforcement/run", h.requireOwner(h.handleEnforcementRunNow))
} }
// handleAPIHealth returns a JSON health status (no auth required). // handleAPIHealth returns a JSON health status (no auth required).
@@ -2993,6 +3000,7 @@ func (h *Handler) handleAdminSettings(w http.ResponseWriter, r *http.Request) {
EmailEnabled: h.mail.IsEnabled(), EmailEnabled: h.mail.IsEnabled(),
MasterKeyPublic: masterPub, MasterKeyPublic: masterPub,
MasterKeyFingerprint: masterFP, MasterKeyFingerprint: masterFP,
EnforcementStatus: h.worker.GetStatus(),
} }
// Check for flash message from query parameters (e.g. after backup restore) // Check for flash message from query parameters (e.g. after backup restore)
@@ -3047,6 +3055,31 @@ func (h *Handler) handleAdminSettings(w http.ResponseWriter, r *http.Request) {
if len(changed) > 0 { if len(changed) > 0 {
h.audit.Log(userID, audit.ActionPasswordPolicyChanged, fmt.Sprintf("Security settings updated: %s", strings.Join(changed, ", ")), clientIP(r)) h.audit.Log(userID, audit.ActionPasswordPolicyChanged, fmt.Sprintf("Security settings updated: %s", strings.Join(changed, ", ")), clientIP(r))
} }
case "enforcement_settings":
// Key enforcement settings
batch := make(map[string]string)
enforceMode := r.FormValue("enforce_mode")
if enforceMode == "" {
enforceMode = "disabled"
}
batch["enforce_mode"] = enforceMode
changed = append(changed, "enforce_mode="+enforceMode)
enforceInterval := r.FormValue("enforce_interval")
if enforceInterval == "" {
enforceInterval = "15"
}
batch["enforce_interval"] = enforceInterval
changed = append(changed, "enforce_interval="+enforceInterval)
if err := h.auth.SetSettingsBatch(batch); err != nil {
logging.Error("Failed to save enforcement settings: %v", err)
http.Redirect(w, r, "/admin/settings?flash_type=danger&flash_msg="+url.QueryEscape("Failed to save enforcement settings: "+err.Error()), http.StatusSeeOther)
return
}
if len(changed) > 0 {
h.audit.Log(userID, audit.ActionEnforcementSettings, fmt.Sprintf("Enforcement settings updated: %s", strings.Join(changed, ", ")), clientIP(r))
}
default: default:
// Application settings (existing behavior) // Application settings (existing behavior)
batch := make(map[string]string) batch := make(map[string]string)
@@ -3124,6 +3157,22 @@ func (h *Handler) handleMasterKeyRegenerate(w http.ResponseWriter, r *http.Reque
http.Redirect(w, r, "/admin/settings?flash_type=success&flash_msg="+url.QueryEscape("System master key successfully regenerated."), http.StatusSeeOther) http.Redirect(w, r, "/admin/settings?flash_type=success&flash_msg="+url.QueryEscape("System master key successfully regenerated."), http.StatusSeeOther)
} }
// handleEnforcementRunNow triggers an immediate key enforcement run (owner only)
func (h *Handler) handleEnforcementRunNow(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Redirect(w, r, "/admin/settings", http.StatusSeeOther)
return
}
userID := h.getUserID(r)
logging.Info("Key enforcement manual run triggered by user_id=%d", userID)
h.audit.Log(userID, audit.ActionEnforcementRun, "Manual key enforcement run triggered", clientIP(r))
h.worker.RunNow()
http.Redirect(w, r, "/admin/settings?flash_type=success&flash_msg="+url.QueryEscape("Key enforcement run started. Check the audit log for results."), http.StatusSeeOther)
}
// --- Cron Job Handlers --- // --- Cron Job Handlers ---
// handleAPICronAssignments returns assignments for a given user as JSON (for AJAX). // handleAPICronAssignments returns assignments for a given user as JSON (for AJAX).

672
internal/worker/worker.go Normal file
View File

@@ -0,0 +1,672 @@
// Keywarden - Centralized SSH Key Management and Deployment
// Copyright (C) 2026 Patrick Asmus (scriptos)
// SPDX-License-Identifier: AGPL-3.0-or-later
package worker
import (
"fmt"
"strings"
"sync"
"time"
"git.techniverse.net/scriptos/keywarden/internal/audit"
"git.techniverse.net/scriptos/keywarden/internal/database"
"git.techniverse.net/scriptos/keywarden/internal/deploy"
"git.techniverse.net/scriptos/keywarden/internal/keys"
"git.techniverse.net/scriptos/keywarden/internal/logging"
"git.techniverse.net/scriptos/keywarden/internal/models"
"git.techniverse.net/scriptos/keywarden/internal/servers"
)
// Mode defines the enforcement behavior
const (
ModeDisabled = "disabled" // no enforcement
ModeMonitor = "monitor" // detect unauthorized keys, log only
ModeEnforce = "enforce" // detect + remove unauthorized keys
)
// DefaultInterval is the default enforcement check interval in minutes
const DefaultInterval = 15
// Service handles the background key enforcement worker
type Service struct {
db *database.DB
deploy *deploy.Service
keys *keys.Service
servers *servers.Service
audit *audit.Service
stopCh chan struct{}
wg sync.WaitGroup
mu sync.Mutex
running bool
}
// NewService creates a new enforcement worker service
func NewService(db *database.DB, deploySvc *deploy.Service, keysSvc *keys.Service, serversSvc *servers.Service, auditSvc *audit.Service) *Service {
return &Service{
db: db,
deploy: deploySvc,
keys: keysSvc,
servers: serversSvc,
audit: auditSvc,
stopCh: make(chan struct{}),
}
}
// Start begins the enforcement worker loop
func (s *Service) Start() {
s.mu.Lock()
if s.running {
s.mu.Unlock()
return
}
s.running = true
s.mu.Unlock()
s.wg.Add(1)
go func() {
defer s.wg.Done()
// Check settings every 60 seconds to see if enforcement is enabled
ticker := time.NewTicker(60 * time.Second)
defer ticker.Stop()
var lastRun time.Time
for {
select {
case <-ticker.C:
mode := s.getMode()
if mode == ModeDisabled {
continue
}
interval := s.getInterval()
if time.Since(lastRun) >= time.Duration(interval)*time.Minute {
s.runEnforcement(mode)
lastRun = time.Now()
}
case <-s.stopCh:
return
}
}
}()
logging.Info("Key enforcement worker started (checks settings every 60s)")
}
// Stop gracefully stops the enforcement worker
func (s *Service) Stop() {
s.mu.Lock()
if !s.running {
s.mu.Unlock()
return
}
s.running = false
s.mu.Unlock()
close(s.stopCh)
s.wg.Wait()
}
// RunNow triggers an immediate enforcement run (e.g. from admin UI)
func (s *Service) RunNow() {
mode := s.getMode()
if mode == ModeDisabled {
logging.Warn("Key enforcement: manual run requested but enforcement is disabled")
return
}
go s.runEnforcement(mode)
}
// getMode reads the enforcement mode from settings
func (s *Service) getMode() string {
var val string
err := s.db.QueryRow(`SELECT value FROM settings WHERE key = 'enforce_mode'`).Scan(&val)
if err != nil || val == "" {
return ModeDisabled
}
switch val {
case ModeMonitor, ModeEnforce:
return val
default:
return ModeDisabled
}
}
// getInterval reads the enforcement interval from settings (in minutes)
func (s *Service) getInterval() int {
var val string
err := s.db.QueryRow(`SELECT value FROM settings WHERE key = 'enforce_interval'`).Scan(&val)
if err != nil || val == "" {
return DefaultInterval
}
var interval int
fmt.Sscanf(val, "%d", &interval)
if interval < 1 {
return DefaultInterval
}
return interval
}
// runEnforcement performs one enforcement cycle across all managed servers
func (s *Service) runEnforcement(mode string) {
logging.Info("Key enforcement: starting run (mode=%s)", mode)
// Get system master key
masterKeyPEM, err := s.keys.GetSystemMasterKeyPrivate()
if err != nil {
logging.Error("Key enforcement: cannot get system master key: %v", err)
return
}
masterKeyPub, err := s.keys.GetSystemMasterKeyPublic()
if err != nil {
logging.Error("Key enforcement: cannot get system master key public: %v", err)
return
}
// Get all servers
allServers, err := s.servers.GetAllServers()
if err != nil {
logging.Error("Key enforcement: failed to get servers: %v", err)
return
}
if len(allServers) == 0 {
logging.Debug("Key enforcement: no servers configured, skipping")
return
}
// Build desired-state map: server_id -> system_user -> []public_key
desiredKeys := s.buildDesiredState(masterKeyPub)
var totalChecked, totalUnauthorized, totalRemoved, totalErrors int
for _, srv := range allServers {
server := srv
// For each server, determine which system users to check
usersToCheck := s.getSystemUsersForServer(server.ID)
// Always check the server's default admin user
if _, exists := usersToCheck[server.Username]; !exists {
usersToCheck[server.Username] = true
}
for systemUser := range usersToCheck {
checked, unauthorized, removed, errs := s.enforceServer(&server, systemUser, masterKeyPEM, masterKeyPub, desiredKeys, mode)
totalChecked += checked
totalUnauthorized += unauthorized
totalRemoved += removed
totalErrors += errs
}
}
// Log summary
summary := fmt.Sprintf("Key enforcement run completed (mode=%s): %d servers checked, %d unauthorized keys found, %d removed, %d errors",
mode, totalChecked, totalUnauthorized, totalRemoved, totalErrors)
logging.Info("%s", summary)
if totalUnauthorized > 0 || totalErrors > 0 {
s.audit.Log(0, audit.ActionEnforcementRun, summary, "worker")
}
// Store last run info in settings
s.setSetting("enforce_last_run", time.Now().UTC().Format(time.RFC3339))
s.setSetting("enforce_last_result", summary)
}
// buildDesiredState builds the complete desired-state map:
//
// server_id -> system_user -> []public_key
//
// Sources of truth (a key is "authorized" if it comes from any of these):
// 1. Access Assignments with desired_state = "present"
// 2. Active Cron Jobs (temporary access) whose keys haven't expired yet
// 3. Direct deployments (via /deploy page) tracked in key_deployments
// 4. The system master key (always authorized on every server+user)
func (s *Service) buildDesiredState(masterKeyPub string) map[int64]map[string][]string {
desired := make(map[int64]map[string][]string)
// Helper to add a key to the desired state (with deduplication)
addKey := func(serverID int64, systemUser, pubKey string) {
if serverID == 0 || systemUser == "" || pubKey == "" {
return
}
if _, ok := desired[serverID]; !ok {
desired[serverID] = make(map[string][]string)
}
pubKey = strings.TrimSpace(pubKey)
for _, existing := range desired[serverID][systemUser] {
if existing == pubKey {
return
}
}
desired[serverID][systemUser] = append(desired[serverID][systemUser], pubKey)
}
// --- Build key lookup: key_id -> public_key ---
allKeys, err := s.keys.GetAllKeys()
if err != nil {
logging.Error("Key enforcement: failed to get all keys: %v", err)
return desired
}
keyMap := make(map[int64]string)
for _, k := range allKeys {
keyMap[k.ID] = strings.TrimSpace(k.PublicKey)
}
// --- Build server lookup: server_id -> Server ---
allSrvs, _ := s.servers.GetAllServers()
srvMap := make(map[int64]*models.Server)
for i := range allSrvs {
srvMap[allSrvs[i].ID] = &allSrvs[i]
}
// --- 1) Access Assignments (desired_state = "present") ---
assignments, err := s.servers.GetAllAssignments()
if err != nil {
logging.Error("Key enforcement: failed to get assignments: %v", err)
} else {
for _, a := range assignments {
if a.DesiredState != "present" {
continue
}
pubKey := keyMap[a.SSHKeyID]
if pubKey == "" {
continue
}
if a.ServerID > 0 {
addKey(a.ServerID, a.SystemUser, pubKey)
}
if a.GroupID > 0 {
members, err := s.servers.GetGroupMembersGlobal(a.GroupID)
if err != nil {
logging.Warn("Key enforcement: failed to resolve group %d: %v", a.GroupID, err)
continue
}
for _, m := range members {
addKey(m.ID, a.SystemUser, pubKey)
}
}
}
logging.Debug("Key enforcement: loaded %d access assignments into desired state", len(assignments))
}
// --- 2) Active Cron Jobs (temporary access, not yet expired) ---
// A cron-deployed key is authorized if:
// - remove_after_min = 0 (permanent) AND the job has executed (last_run IS NOT NULL)
// - remove_after_min > 0 AND last_run + remove_after_min > NOW() (not yet expired)
cronCount := s.addCronJobKeys(addKey, keyMap, srvMap)
logging.Debug("Key enforcement: loaded %d active cron job deployments into desired state", cronCount)
// --- 3) Direct deployments (via /deploy page) ---
// These are tracked in key_deployments. For each key+server pair, the latest
// successful deploy (not removal) authorizes the key for the server's admin user.
deployCount := s.addDirectDeployKeys(addKey, keyMap, srvMap)
logging.Debug("Key enforcement: loaded %d direct deployments into desired state", deployCount)
// --- 4) System master key (always authorized everywhere) ---
masterPub := strings.TrimSpace(masterKeyPub)
for _, srv := range allSrvs {
// Master key on every server's admin user
addKey(srv.ID, srv.Username, masterPub)
// Master key on every system user that has desired keys
if users, ok := desired[srv.ID]; ok {
for sysUser := range users {
addKey(srv.ID, sysUser, masterPub)
}
}
}
return desired
}
// addCronJobKeys queries cron_jobs for active temporary deployments and adds
// their keys to the desired state. Returns the number of active cron deployments found.
func (s *Service) addCronJobKeys(addKey func(int64, string, string), keyMap map[int64]string, srvMap map[int64]*models.Server) int {
// Query cron jobs whose deployed keys should still be on the server:
// - Job has executed at least once (last_run IS NOT NULL)
// - Either permanent (remove_after_min = 0) or not yet expired
// - Job status indicates it has executed (not just created)
rows, err := s.db.Query(
`SELECT cj.ssh_key_id, cj.server_id, cj.group_id, cj.system_user
FROM cron_jobs cj
WHERE cj.last_run IS NOT NULL
AND cj.status IN ('done', 'active', 'running')
AND (
cj.remove_after_min = 0
OR datetime(cj.last_run, '+' || cj.remove_after_min || ' minutes') > datetime('now')
)`)
if err != nil {
logging.Warn("Key enforcement: failed to query active cron jobs: %v", err)
return 0
}
defer rows.Close()
var count int
for rows.Next() {
var keyID, serverID, groupID int64
var systemUser string
if err := rows.Scan(&keyID, &serverID, &groupID, &systemUser); err != nil {
continue
}
pubKey := keyMap[keyID]
if pubKey == "" {
continue
}
if serverID > 0 {
if systemUser != "" {
addKey(serverID, systemUser, pubKey)
} else if srv, ok := srvMap[serverID]; ok {
// No system user specified → deployed to server's admin user
addKey(serverID, srv.Username, pubKey)
}
count++
}
if groupID > 0 {
members, err := s.servers.GetGroupMembersGlobal(groupID)
if err != nil {
continue
}
for _, m := range members {
if systemUser != "" {
addKey(m.ID, systemUser, pubKey)
} else {
addKey(m.ID, m.Username, pubKey)
}
}
count++
}
}
return count
}
// addDirectDeployKeys queries key_deployments for successful direct deployments
// (via /deploy page) and adds their keys to the desired state.
// For each key+server pair, the most recent entry determines if the key is still deployed.
// Direct deploys always target the server's configured admin user.
func (s *Service) addDirectDeployKeys(addKey func(int64, string, string), keyMap map[int64]string, srvMap map[int64]*models.Server) int {
// Get the latest deployment status for each key+server combination.
// A key is considered deployed if the latest entry contains "deployed" (not "removed").
rows, err := s.db.Query(
`SELECT kd.ssh_key_id, kd.server_id, kd.message
FROM key_deployments kd
INNER JOIN (
SELECT ssh_key_id, server_id, MAX(id) as max_id
FROM key_deployments
WHERE status = 'success'
GROUP BY ssh_key_id, server_id
) latest ON kd.id = latest.max_id
WHERE kd.message LIKE '%deployed%'`)
if err != nil {
logging.Warn("Key enforcement: failed to query direct deployments: %v", err)
return 0
}
defer rows.Close()
var count int
for rows.Next() {
var keyID, serverID int64
var message string
if err := rows.Scan(&keyID, &serverID, &message); err != nil {
continue
}
pubKey := keyMap[keyID]
if pubKey == "" {
continue
}
srv, ok := srvMap[serverID]
if !ok {
continue
}
// Determine the system user from the deployment message
// DeployKeyToUser logs: "key deployed to user 'xxx'"
// DeployKey logs: "key deployed successfully" (→ server's admin user)
systemUser := srv.Username
if idx := strings.Index(message, "to user '"); idx >= 0 {
rest := message[idx+len("to user '"):]
if endIdx := strings.Index(rest, "'"); endIdx >= 0 {
systemUser = rest[:endIdx]
}
}
addKey(serverID, systemUser, pubKey)
count++
}
return count
}
// getSystemUsersForServer returns all system users that should be checked on a server.
// This includes users from:
// 1. Access Assignments (direct + group)
// 2. Active Cron Jobs (direct + group)
func (s *Service) getSystemUsersForServer(serverID int64) map[string]bool {
users := make(map[string]bool)
// --- 1a) Direct access assignments ---
rows, err := s.db.Query(
`SELECT DISTINCT system_user FROM access_assignments WHERE server_id = ? AND desired_state = 'present'`, serverID)
if err == nil {
for rows.Next() {
var u string
if rows.Scan(&u) == nil && u != "" {
users[u] = true
}
}
rows.Close()
}
// --- 1b) Group access assignments ---
groupRows, err := s.db.Query(
`SELECT DISTINCT a.system_user FROM access_assignments a
JOIN server_group_members sgm ON a.group_id = sgm.group_id
WHERE sgm.server_id = ? AND a.desired_state = 'present' AND a.group_id > 0`, serverID)
if err == nil {
for groupRows.Next() {
var u string
if groupRows.Scan(&u) == nil && u != "" {
users[u] = true
}
}
groupRows.Close()
}
// --- 2a) Direct cron jobs (active temporary access) ---
cronRows, err := s.db.Query(
`SELECT DISTINCT cj.system_user FROM cron_jobs cj
WHERE cj.server_id = ?
AND cj.last_run IS NOT NULL
AND cj.status IN ('done', 'active', 'running')
AND cj.system_user != ''
AND (
cj.remove_after_min = 0
OR datetime(cj.last_run, '+' || cj.remove_after_min || ' minutes') > datetime('now')
)`, serverID)
if err == nil {
for cronRows.Next() {
var u string
if cronRows.Scan(&u) == nil && u != "" {
users[u] = true
}
}
cronRows.Close()
}
// --- 2b) Group cron jobs ---
cronGroupRows, err := s.db.Query(
`SELECT DISTINCT cj.system_user FROM cron_jobs cj
JOIN server_group_members sgm ON cj.group_id = sgm.group_id
WHERE sgm.server_id = ?
AND cj.last_run IS NOT NULL
AND cj.status IN ('done', 'active', 'running')
AND cj.system_user != ''
AND cj.group_id > 0
AND (
cj.remove_after_min = 0
OR datetime(cj.last_run, '+' || cj.remove_after_min || ' minutes') > datetime('now')
)`, serverID)
if err == nil {
for cronGroupRows.Next() {
var u string
if cronGroupRows.Scan(&u) == nil && u != "" {
users[u] = true
}
}
cronGroupRows.Close()
}
return users
}
// enforceServer checks and optionally enforces key state for one server+user combination
func (s *Service) enforceServer(server *models.Server, systemUser string, masterKeyPEM []byte, masterKeyPub string, desiredKeys map[int64]map[string][]string, mode string) (checked, unauthorized, removed, errors int) {
checked = 1
// Read current authorized_keys from the server
currentKeys, err := s.deploy.ReadAuthorizedKeys(server, masterKeyPEM, systemUser)
if err != nil {
logging.Warn("Key enforcement: failed to read keys from %s@%s:%d (user=%s): %v",
server.Username, server.Hostname, server.Port, systemUser, err)
errors = 1
return
}
// Get desired keys for this server+user
var desired []string
if serverUsers, ok := desiredKeys[server.ID]; ok {
if keys, ok := serverUsers[systemUser]; ok {
desired = keys
}
}
// Always include the master key
masterPub := strings.TrimSpace(masterKeyPub)
hasMaster := false
for _, k := range desired {
if k == masterPub {
hasMaster = true
break
}
}
if !hasMaster {
desired = append(desired, masterPub)
}
// Build set of desired key fingerprints/content for comparison
desiredSet := make(map[string]bool)
for _, k := range desired {
desiredSet[normalizeKey(k)] = true
}
// Find unauthorized keys
var unauthorizedKeys []string
for _, currentKey := range currentKeys {
normalized := normalizeKey(currentKey)
if normalized == "" {
continue
}
if !desiredSet[normalized] {
unauthorizedKeys = append(unauthorizedKeys, currentKey)
}
}
unauthorized = len(unauthorizedKeys)
if unauthorized == 0 {
logging.Debug("Key enforcement: %s@%s (user=%s): all %d keys authorized",
server.Username, server.Hostname, systemUser, len(currentKeys))
return
}
// Log the unauthorized keys
keySnippets := make([]string, 0, len(unauthorizedKeys))
for _, k := range unauthorizedKeys {
snippet := k
if len(snippet) > 80 {
snippet = snippet[:80] + "..."
}
keySnippets = append(keySnippets, snippet)
}
detail := fmt.Sprintf("Server %s (%s:%d), user '%s': %d unauthorized key(s) found: %s",
server.Name, server.Hostname, server.Port, systemUser,
unauthorized, strings.Join(keySnippets, "; "))
if mode == ModeMonitor {
logging.Warn("Key enforcement [MONITOR]: %s", detail)
s.audit.Log(0, audit.ActionEnforcementDrift, detail, "worker")
return
}
// Mode: enforce — replace authorized_keys with only desired keys
logging.Warn("Key enforcement [ENFORCE]: %s — removing unauthorized keys", detail)
s.audit.Log(0, audit.ActionEnforcementDrift, detail, "worker")
if err := s.deploy.WriteAuthorizedKeys(server, masterKeyPEM, systemUser, desired); err != nil {
logging.Error("Key enforcement: failed to write authorized_keys for %s@%s (user=%s): %v",
server.Username, server.Hostname, systemUser, err)
s.audit.Log(0, audit.ActionEnforcementFailed,
fmt.Sprintf("Failed to enforce keys on %s (%s:%d) user '%s': %v", server.Name, server.Hostname, server.Port, systemUser, err),
"worker")
errors = 1
return
}
removed = unauthorized
s.audit.Log(0, audit.ActionEnforcementApplied,
fmt.Sprintf("Enforced authorized_keys on %s (%s:%d) user '%s': removed %d unauthorized key(s)",
server.Name, server.Hostname, server.Port, systemUser, removed),
"worker")
return
}
// normalizeKey normalizes a public key line for comparison (strips comments and whitespace variations)
func normalizeKey(key string) string {
key = strings.TrimSpace(key)
if key == "" || strings.HasPrefix(key, "#") {
return ""
}
// SSH public keys have format: type base64data [comment]
// We compare type + base64data only (ignore the comment)
parts := strings.Fields(key)
if len(parts) >= 2 {
return parts[0] + " " + parts[1]
}
return key
}
// setSetting writes a value to the settings table (upsert)
func (s *Service) setSetting(key, value string) {
s.db.Exec(
`INSERT INTO settings (key, value, updated_at) VALUES (?, ?, CURRENT_TIMESTAMP)
ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = CURRENT_TIMESTAMP`,
key, value,
)
}
// GetStatus returns the current enforcement worker status for display
func (s *Service) GetStatus() map[string]string {
status := make(map[string]string)
var val string
if err := s.db.QueryRow(`SELECT value FROM settings WHERE key = 'enforce_mode'`).Scan(&val); err == nil {
status["mode"] = val
} else {
status["mode"] = ModeDisabled
}
if err := s.db.QueryRow(`SELECT value FROM settings WHERE key = 'enforce_interval'`).Scan(&val); err == nil {
status["interval"] = val
} else {
status["interval"] = fmt.Sprintf("%d", DefaultInterval)
}
if err := s.db.QueryRow(`SELECT value FROM settings WHERE key = 'enforce_last_run'`).Scan(&val); err == nil {
status["last_run"] = val
}
if err := s.db.QueryRow(`SELECT value FROM settings WHERE key = 'enforce_last_result'`).Scan(&val); err == nil {
status["last_result"] = val
}
return status
}

View File

@@ -284,6 +284,87 @@
</div> </div>
</div> </div>
<!-- Key Enforcement (Bastillion-Style) -->
<div class="col-12">
<div class="card">
<div class="card-header">
<h3 class="card-title"><i class="ti ti-shield-check"></i> Key Enforcement</h3>
</div>
<div class="card-body">
<div class="alert alert-info">
<div class="d-flex">
<div><i class="ti ti-info-circle icon alert-icon"></i></div>
<div>
<h4 class="alert-title">Enforced Key Management</h4>
<div class="text-secondary">
When enabled, Keywarden periodically connects to all managed servers and verifies that only
authorized SSH keys (managed by Keywarden + the system master key) are present in
<code>authorized_keys</code>. Unauthorized keys are detected and optionally removed automatically.
<br><br>
<strong>Monitor mode:</strong> Detects unauthorized keys and logs them in the audit log, but does not remove them.<br>
<strong>Enforce mode:</strong> Detects unauthorized keys and <em>removes them automatically</em>, keeping only Keywarden-managed keys.
</div>
</div>
</div>
</div>
<form action="/admin/settings" method="post">
<input type="hidden" name="form_type" value="enforcement_settings">
<div class="row mb-3">
<div class="col-md-4 mb-3">
<label class="form-label">Enforcement Mode</label>
<select name="enforce_mode" class="form-select">
<option value="disabled" {{if or (not .EnforcementStatus) (eq (index .EnforcementStatus "mode") "disabled")}}selected{{end}}>Disabled</option>
<option value="monitor" {{if and .EnforcementStatus (eq (index .EnforcementStatus "mode") "monitor")}}selected{{end}}>Monitor (detect only)</option>
<option value="enforce" {{if and .EnforcementStatus (eq (index .EnforcementStatus "mode") "enforce")}}selected{{end}}>Enforce (detect &amp; remove)</option>
</select>
<small class="form-hint">Choose how Keywarden handles unauthorized keys on your servers.</small>
</div>
<div class="col-md-4 mb-3">
<label class="form-label">Check Interval (minutes)</label>
<input type="number" name="enforce_interval" class="form-control"
value="{{if and .EnforcementStatus (index .EnforcementStatus "interval")}}{{index .EnforcementStatus "interval"}}{{else}}15{{end}}"
min="1" max="1440" placeholder="15">
<small class="form-hint">How often Keywarden checks the servers (11440 minutes).</small>
</div>
</div>
<div class="form-footer">
<button type="submit" class="btn btn-primary">
<i class="ti ti-device-floppy"></i> Save Enforcement Settings
</button>
</div>
</form>
{{if and .EnforcementStatus (index .EnforcementStatus "last_run")}}
<hr class="my-4">
<h4 class="mb-3"><i class="ti ti-history"></i> Last Enforcement Run</h4>
<div class="datagrid mb-3">
<div class="datagrid-item">
<div class="datagrid-title">Last Run</div>
<div class="datagrid-content">{{index .EnforcementStatus "last_run"}}</div>
</div>
<div class="datagrid-item">
<div class="datagrid-title">Result</div>
<div class="datagrid-content">{{index .EnforcementStatus "last_result"}}</div>
</div>
</div>
{{end}}
{{if and .EnforcementStatus (ne (index .EnforcementStatus "mode") "disabled")}}
<hr class="my-4">
<h4 class="mb-3"><i class="ti ti-player-play"></i> Manual Run</h4>
<form action="/admin/enforcement/run" method="post" onsubmit="return confirm('Start a key enforcement run now? This will connect to all managed servers.');">
<button type="submit" class="btn btn-warning">
<i class="ti ti-player-play"></i> Run Enforcement Now
</button>
<small class="form-hint d-inline-block ms-2">Trigger an immediate enforcement check on all servers.</small>
</form>
{{end}}
</div>
</div>
</div>
<!-- Backup & Restore --> <!-- Backup & Restore -->
<div class="col-12"> <div class="col-12">
<div class="card"> <div class="card">