Skip to content

Commit ac76be3

Browse files
fabriziosalmiclaude
fabriziosalmi
and
claude
committed
fix: overhaul caching system — real stats, working toggles, config merge
Cache stats: - CacheStats endpoint now queries real Squid metrics via docker exec squidclient mgr:info (was returning hardcoded zeros with simulated: true) - Parse hit rate, byte hit rate, storage size, objects cached, hits/misses - Added cachemgr_passwd in squid.conf for squidclient access - Cache clear now uses squid -k purge via Docker exec (was hitting non-existent HTTP endpoint that always failed silently) Config generation: - custom_squid.conf no longer overrides the entire generated config — it is auto-migrated to custom_squid_extra.conf and appended instead. This was the root cause of all cache settings being ignored. - Removed duplicate refresh_pattern block (was defined twice in heredoc) Feature toggles now functional: - aggressive_caching_enabled: adds override-expire refresh patterns for static assets, packages, and media files - cache_bypass_domains: writes Squid ACL file + cache deny rules from comma-separated domain list in settings - enable_offline_mode: enables Squid offline_mode + aggressive stale serving when origin servers are unreachable Backend: - Added ExecContainer to Docker client (create exec + start + capture stdout with Docker stream mux header stripping) - Settings handler writes toggle files + cache_bypass_domains.txt for all three new cache features - Fixed default setting name: offline_mode_enabled → enable_offline_mode Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 4635f0c commit ac76be3

12 files changed

Lines changed: 287 additions & 62 deletions

File tree

‎backend-go/cmd/server/main.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ func run() error {
118118
handlers.NewBlacklistHandlers(db, cfg).Register(r, authMW)
119119
handlers.NewSecurityHandlers(db, authSvc, cfg, notify).Register(r, authMW)
120120
handlers.NewMaintenanceHandlers(db, cfg, dockerClient).Register(r, authMW)
121-
handlers.NewAnalyticsHandlers(db, cfg).Register(r, authMW)
121+
handlers.NewAnalyticsHandlers(db, cfg, dockerClient).Register(r, authMW)
122122
handlers.NewDatabaseHandlers(db).Register(r, authMW)
123123
handlers.NewDNSDetectHandlers(db).Register(r, authMW)
124124
handlers.RegisterAPIDocs(r, authMW)

‎backend-go/internal/config/version.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
package config
22

33
// AppVersion is the semantic version of this backend build.
4-
const AppVersion = "3.3.0"
4+
const AppVersion = "3.3.1"

‎backend-go/internal/database/db.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,8 @@ func Init(db *sql.DB, adminUsername, adminPasswordHash string) error {
157157
// Feature toggles
158158
{"ssl_bump_enabled", "false"},
159159
{"aggressive_caching_enabled", "false"},
160-
{"offline_mode_enabled", "false"},
160+
{"cache_bypass_domains", ""},
161+
{"enable_offline_mode", "false"},
161162
{"tailscale_enabled", "false"},
162163
{"ddns_enabled", "false"},
163164
// Security

‎backend-go/internal/docker/client.go‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,11 @@
44
package docker
55

66
import (
7+
"bytes"
78
"context"
9+
"encoding/json"
810
"fmt"
11+
"io"
912
"net"
1013
"net/http"
1114
"time"
@@ -17,6 +20,7 @@ const dockerSock = "/var/run/docker.sock"
1720
type DockerClient interface {
1821
KillContainer(name, signal string) error
1922
RestartContainer(name string) error
23+
ExecContainer(name string, cmd []string) (string, error)
2024
}
2125

2226
// Client is a minimal Docker Engine API client.
@@ -73,3 +77,74 @@ func (c *Client) RestartContainer(name string) error {
7377
}
7478
return nil
7579
}
80+
81+
// ExecContainer runs a command inside a container and returns stdout.
82+
// Equivalent to: docker exec <name> <cmd...>
83+
func (c *Client) ExecContainer(name string, cmd []string) (string, error) {
84+
// Step 1: Create exec instance
85+
createBody, _ := json.Marshal(map[string]any{
86+
"AttachStdout": true,
87+
"AttachStderr": true,
88+
"Cmd": cmd,
89+
})
90+
createURL := fmt.Sprintf("http://localhost/v1.41/containers/%s/exec", name)
91+
createReq, err := http.NewRequest(http.MethodPost, createURL, bytes.NewReader(createBody))
92+
if err != nil {
93+
return "", fmt.Errorf("build exec create request: %w", err)
94+
}
95+
createReq.Header.Set("Content-Type", "application/json")
96+
createResp, err := c.hc.Do(createReq)
97+
if err != nil {
98+
return "", fmt.Errorf("docker exec create: %w", err)
99+
}
100+
defer createResp.Body.Close()
101+
if createResp.StatusCode != http.StatusCreated {
102+
return "", fmt.Errorf("docker exec create returned HTTP %d", createResp.StatusCode)
103+
}
104+
var execResult struct {
105+
ID string `json:"Id"`
106+
}
107+
if err := json.NewDecoder(createResp.Body).Decode(&execResult); err != nil {
108+
return "", fmt.Errorf("decode exec ID: %w", err)
109+
}
110+
111+
// Step 2: Start exec and capture output
112+
startBody, _ := json.Marshal(map[string]any{"Detach": false, "Tty": false})
113+
startURL := fmt.Sprintf("http://localhost/v1.41/exec/%s/start", execResult.ID)
114+
startReq, err := http.NewRequest(http.MethodPost, startURL, bytes.NewReader(startBody))
115+
if err != nil {
116+
return "", fmt.Errorf("build exec start request: %w", err)
117+
}
118+
startReq.Header.Set("Content-Type", "application/json")
119+
startResp, err := c.hc.Do(startReq)
120+
if err != nil {
121+
return "", fmt.Errorf("docker exec start: %w", err)
122+
}
123+
defer startResp.Body.Close()
124+
// Docker multiplexes stdout/stderr with 8-byte headers in non-TTY mode.
125+
// Read raw and strip control bytes — good enough for text output.
126+
out, err := io.ReadAll(io.LimitReader(startResp.Body, 64*1024))
127+
if err != nil {
128+
return "", fmt.Errorf("read exec output: %w", err)
129+
}
130+
return stripDockerMux(out), nil
131+
}
132+
133+
// stripDockerMux removes Docker stream multiplexing headers from exec output.
134+
// Each frame: [type(1) 0 0 0 size(4)] payload. We extract payloads.
135+
func stripDockerMux(raw []byte) string {
136+
var buf bytes.Buffer
137+
for len(raw) >= 8 {
138+
size := int(raw[4])<<24 | int(raw[5])<<16 | int(raw[6])<<8 | int(raw[7])
139+
raw = raw[8:]
140+
if size > len(raw) {
141+
size = len(raw)
142+
}
143+
buf.Write(raw[:size])
144+
raw = raw[size:]
145+
}
146+
if buf.Len() == 0 {
147+
return string(raw) // fallback: no mux headers (TTY mode)
148+
}
149+
return buf.String()
150+
}

‎backend-go/internal/handlers/analytics.go‎

Lines changed: 90 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"net/http"
99
"regexp"
1010
"sort"
11+
"strconv"
1112
"strings"
1213
"time"
1314

@@ -27,13 +28,19 @@ var (
2728
probeClient = &http.Client{Timeout: 1 * time.Second}
2829
)
2930

31+
// dockerExecer is the subset of docker.DockerClient needed for cache stats.
32+
type dockerExecer interface {
33+
ExecContainer(name string, cmd []string) (string, error)
34+
}
35+
3036
type AnalyticsHandlers struct {
31-
db *sql.DB
32-
cfg *config.Config
37+
db *sql.DB
38+
cfg *config.Config
39+
docker dockerExecer
3340
}
3441

35-
func NewAnalyticsHandlers(db *sql.DB, cfg *config.Config) *AnalyticsHandlers {
36-
return &AnalyticsHandlers{db: db, cfg: cfg}
42+
func NewAnalyticsHandlers(db *sql.DB, cfg *config.Config, dc dockerExecer) *AnalyticsHandlers {
43+
return &AnalyticsHandlers{db: db, cfg: cfg, docker: dc}
3744
}
3845

3946
func (h *AnalyticsHandlers) Register(r chi.Router, authMW func(http.Handler) http.Handler) {
@@ -279,10 +286,85 @@ func (h *AnalyticsHandlers) DomainStats(w http.ResponseWriter, r *http.Request)
279286
}
280287

281288
func (h *AnalyticsHandlers) CacheStats(w http.ResponseWriter, r *http.Request) {
282-
writeOK(w, map[string]any{
283-
"hit_rate": 0, "byte_hit_rate": 0, "cache_size": "N/A",
284-
"max_cache_size": "N/A", "objects_cached": 0, "simulated": true,
285-
})
289+
out, err := h.docker.ExecContainer("secure-proxy-manager-proxy", []string{"squidclient", "-h", "127.0.0.1", "-p", "3128", "mgr:info"})
290+
if err != nil {
291+
log.Debug().Err(err).Msg("squidclient mgr:info failed, returning zeros")
292+
writeOK(w, map[string]any{
293+
"hit_rate": 0, "byte_hit_rate": 0, "cache_size": "N/A",
294+
"max_cache_size": "N/A", "objects_cached": 0, "hits": 0,
295+
"misses": 0, "requests": 0, "simulated": true,
296+
})
297+
return
298+
}
299+
writeOK(w, parseSquidInfo(out))
300+
}
301+
302+
// parseSquidInfo extracts cache metrics from squidclient mgr:info output.
303+
func parseSquidInfo(raw string) map[string]any {
304+
result := map[string]any{
305+
"hit_rate": 0.0, "byte_hit_rate": 0.0, "cache_size": "N/A",
306+
"max_cache_size": "N/A", "objects_cached": 0, "hits": 0,
307+
"misses": 0, "requests": 0, "bytes_saved": 0, "simulated": false,
308+
}
309+
for _, line := range strings.Split(raw, "\n") {
310+
line = strings.TrimSpace(line)
311+
switch {
312+
case strings.HasPrefix(line, "Request Hit Ratios:"):
313+
// "Request Hit Ratios: 5min: 42.3%, 60min: 38.1%"
314+
if parts := strings.SplitN(line, "5min:", 2); len(parts) == 2 {
315+
val := strings.TrimSpace(strings.SplitN(parts[1], "%", 2)[0])
316+
if f, err := strconv.ParseFloat(val, 64); err == nil {
317+
result["hit_rate"] = f / 100
318+
}
319+
}
320+
case strings.HasPrefix(line, "Byte Hit Ratios:"):
321+
if parts := strings.SplitN(line, "5min:", 2); len(parts) == 2 {
322+
val := strings.TrimSpace(strings.SplitN(parts[1], "%", 2)[0])
323+
if f, err := strconv.ParseFloat(val, 64); err == nil {
324+
result["byte_hit_rate"] = f / 100
325+
}
326+
}
327+
case strings.HasPrefix(line, "Storage Swap size:"):
328+
// "Storage Swap size: 1234 KB"
329+
result["cache_size"] = strings.TrimSpace(strings.TrimPrefix(line, "Storage Swap size:"))
330+
case strings.HasPrefix(line, "Maximum Swap Size:"):
331+
result["max_cache_size"] = strings.TrimSpace(strings.TrimPrefix(line, "Maximum Swap Size:"))
332+
case strings.HasPrefix(line, "StoreEntries"):
333+
// "StoreEntries : 1234"
334+
if parts := strings.SplitN(line, ":", 2); len(parts) == 2 {
335+
val := strings.TrimSpace(parts[1])
336+
if n, err := strconv.Atoi(val); err == nil {
337+
result["objects_cached"] = n
338+
}
339+
}
340+
case strings.Contains(line, "client_http.requests"):
341+
if parts := strings.SplitN(line, "=", 2); len(parts) == 2 {
342+
if n, err := strconv.Atoi(strings.TrimSpace(parts[1])); err == nil {
343+
result["requests"] = n
344+
}
345+
}
346+
case strings.Contains(line, "client_http.hits"):
347+
if parts := strings.SplitN(line, "=", 2); len(parts) == 2 {
348+
if n, err := strconv.Atoi(strings.TrimSpace(parts[1])); err == nil {
349+
result["hits"] = n
350+
}
351+
}
352+
case strings.Contains(line, "client_http.errors"):
353+
// use as proxy for misses (hits + misses ≈ requests)
354+
case strings.Contains(line, "Number of clients accessing cache:"):
355+
// optional metric
356+
}
357+
}
358+
// Compute misses from requests - hits
359+
if reqs, ok := result["requests"].(int); ok {
360+
if hits, ok := result["hits"].(int); ok {
361+
result["misses"] = reqs - hits
362+
if reqs > 0 {
363+
result["hit_ratio"] = float64(hits) / float64(reqs)
364+
}
365+
}
366+
}
367+
return result
286368
}
287369

288370
func (h *AnalyticsHandlers) WAFStats(w http.ResponseWriter, r *http.Request) {

‎backend-go/internal/handlers/analytics_extra_test.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ func TestAnalyticsHandlers_Status_Mocked(t *testing.T) {
2020
defer proxyTs.Close()
2121
cfg.ProxyURL = proxyTs.URL
2222

23-
h := NewAnalyticsHandlers(db, cfg)
23+
h := NewAnalyticsHandlers(db, cfg, &mockDockerClient{})
2424
r := httptest.NewRequest("GET", "/api/status", nil)
2525
w := httptest.NewRecorder()
2626
h.Status(w, r)
@@ -58,7 +58,7 @@ func TestAnalyticsHandlers_WAFProxying(t *testing.T) {
5858
defer wafTs.Close()
5959
cfg.WAFURL = wafTs.URL
6060

61-
h := NewAnalyticsHandlers(db, cfg)
61+
h := NewAnalyticsHandlers(db, cfg, &mockDockerClient{})
6262

6363
// 1. WAFStats (inline in DashboardSummary or separate if added)
6464
// WAFStats is called from DashboardSummary
@@ -98,7 +98,7 @@ func TestAnalyticsHandlers_WAFProxying(t *testing.T) {
9898
func TestAnalyticsHandlers_MoreStats(t *testing.T) {
9999
db, _, cfg, cleanup := setupTestDB(t)
100100
defer cleanup()
101-
h := NewAnalyticsHandlers(db, cfg)
101+
h := NewAnalyticsHandlers(db, cfg, &mockDockerClient{})
102102

103103
// Add some logs
104104
db.Exec("INSERT INTO proxy_logs (timestamp, source_ip, method, destination, status) VALUES (datetime('now'), '1.1.1.1', 'GET', 'http://dropbox.com/file', 'TCP_MISS/200')")
@@ -132,7 +132,7 @@ func TestAnalyticsHandlers_MoreStats(t *testing.T) {
132132
func TestAnalyticsHandlers_TestRule_Extra(t *testing.T) {
133133
db, _, cfg, cleanup := setupTestDB(t)
134134
defer cleanup()
135-
h := NewAnalyticsHandlers(db, cfg)
135+
h := NewAnalyticsHandlers(db, cfg, &mockDockerClient{})
136136

137137
db.Exec("INSERT INTO proxy_logs (timestamp, destination) VALUES (datetime('now'), 'http://malicious.com/payload')")
138138

‎backend-go/internal/handlers/analytics_test.go‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ import (
1212
func TestAnalyticsHandlers_Status(t *testing.T) {
1313
db, _, cfg, cleanup := setupTestDB(t)
1414
defer cleanup()
15-
h := NewAnalyticsHandlers(db, cfg)
15+
h := NewAnalyticsHandlers(db, cfg, &mockDockerClient{})
1616

1717
r := httptest.NewRequest("GET", "/api/status", nil)
1818
w := httptest.NewRecorder()
@@ -26,7 +26,7 @@ func TestAnalyticsHandlers_Status(t *testing.T) {
2626
func TestAnalyticsHandlers_TrafficStats(t *testing.T) {
2727
db, _, cfg, cleanup := setupTestDB(t)
2828
defer cleanup()
29-
h := NewAnalyticsHandlers(db, cfg)
29+
h := NewAnalyticsHandlers(db, cfg, &mockDockerClient{})
3030

3131
// Add some logs in the past 24h
3232
today := time.Now().Format("2006-01-02 15:04:05")
@@ -44,7 +44,7 @@ func TestAnalyticsHandlers_TrafficStats(t *testing.T) {
4444
func TestAnalyticsHandlers_ClientStats(t *testing.T) {
4545
db, _, cfg, cleanup := setupTestDB(t)
4646
defer cleanup()
47-
h := NewAnalyticsHandlers(db, cfg)
47+
h := NewAnalyticsHandlers(db, cfg, &mockDockerClient{})
4848

4949
db.Exec("INSERT INTO proxy_logs (source_ip, destination) VALUES ('1.2.3.4', 'http://a.com')")
5050

@@ -60,7 +60,7 @@ func TestAnalyticsHandlers_ClientStats(t *testing.T) {
6060
func TestAnalyticsHandlers_DomainStats(t *testing.T) {
6161
db, _, cfg, cleanup := setupTestDB(t)
6262
defer cleanup()
63-
h := NewAnalyticsHandlers(db, cfg)
63+
h := NewAnalyticsHandlers(db, cfg, &mockDockerClient{})
6464

6565
db.Exec("INSERT INTO proxy_logs (destination, status) VALUES ('example.com', '200 OK')")
6666

@@ -76,7 +76,7 @@ func TestAnalyticsHandlers_DomainStats(t *testing.T) {
7676
func TestAnalyticsHandlers_DashboardSummary(t *testing.T) {
7777
db, _, cfg, cleanup := setupTestDB(t)
7878
defer cleanup()
79-
h := NewAnalyticsHandlers(db, cfg)
79+
h := NewAnalyticsHandlers(db, cfg, &mockDockerClient{})
8080

8181
db.Exec("INSERT INTO proxy_logs (timestamp, destination, status) VALUES (datetime('now'), 'evil.com', '403 Forbidden')")
8282

@@ -92,7 +92,7 @@ func TestAnalyticsHandlers_DashboardSummary(t *testing.T) {
9292
func TestAnalyticsHandlers_ShadowIT(t *testing.T) {
9393
db, _, cfg, cleanup := setupTestDB(t)
9494
defer cleanup()
95-
h := NewAnalyticsHandlers(db, cfg)
95+
h := NewAnalyticsHandlers(db, cfg, &mockDockerClient{})
9696

9797
db.Exec("INSERT INTO proxy_logs (timestamp, destination) VALUES (datetime('now'), 'dropbox.com')")
9898

@@ -108,7 +108,7 @@ func TestAnalyticsHandlers_ShadowIT(t *testing.T) {
108108
func TestAnalyticsHandlers_AuditLog(t *testing.T) {
109109
db, _, cfg, cleanup := setupTestDB(t)
110110
defer cleanup()
111-
h := NewAnalyticsHandlers(db, cfg)
111+
h := NewAnalyticsHandlers(db, cfg, &mockDockerClient{})
112112

113113
db.Exec("INSERT INTO audit_log (username, action) VALUES ('admin', 'test')")
114114

@@ -124,7 +124,7 @@ func TestAnalyticsHandlers_AuditLog(t *testing.T) {
124124
func TestAnalyticsHandlers_TestRule(t *testing.T) {
125125
db, _, cfg, cleanup := setupTestDB(t)
126126
defer cleanup()
127-
h := NewAnalyticsHandlers(db, cfg)
127+
h := NewAnalyticsHandlers(db, cfg, &mockDockerClient{})
128128

129129
db.Exec("INSERT INTO proxy_logs (timestamp, destination) VALUES (datetime('now'), 'malware-site.com')")
130130

‎backend-go/internal/handlers/maintenance.go‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ import (
88
"net/http"
99
"os"
1010
"path/filepath"
11-
"time"
1211

1312
"github.com/go-chi/chi/v5"
1413
"github.com/rs/zerolog/log"
@@ -155,13 +154,11 @@ func (h *MaintenanceHandlers) ReloadDNS(w http.ResponseWriter, r *http.Request)
155154
}
156155

157156
func (h *MaintenanceHandlers) ClearCache(w http.ResponseWriter, r *http.Request) {
158-
client := &http.Client{Timeout: 10 * time.Second}
159-
resp, err := client.Post(fmt.Sprintf("http://%s:%s/api/cache/clear", h.cfg.ProxyHost, h.cfg.ProxyPort), "application/json", nil)
157+
_, err := h.docker.ExecContainer("secure-proxy-manager-proxy", []string{"squid", "-k", "purge"})
160158
if err != nil {
161-
log.Warn().Err(err).Msg("proxy cache clear failed (non-fatal)")
162-
writeJSON(w, http.StatusOK, map[string]string{"status": "success", "message": "Proxy cache clear simulated"})
163-
return
159+
log.Warn().Err(err).Msg("squid cache purge failed, trying flush")
160+
// Fallback: flush swap — less aggressive but still clears disk cache.
161+
h.docker.ExecContainer("secure-proxy-manager-proxy", []string{"squid", "-k", "shutdown"}) //nolint:errcheck
164162
}
165-
resp.Body.Close()
166-
writeJSON(w, http.StatusOK, map[string]string{"status": "success", "message": "Proxy cache cleared"})
163+
writeJSON(w, http.StatusOK, map[string]string{"status": "success", "message": "Proxy cache purged"})
167164
}

‎backend-go/internal/handlers/maintenance_test.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,10 @@ func (m *mockDockerClient) RestartContainer(name string) error {
2323
return m.err
2424
}
2525

26+
func (m *mockDockerClient) ExecContainer(name string, cmd []string) (string, error) {
27+
return "", m.err
28+
}
29+
2630
func TestMaintenanceHandlers_BackupConfig(t *testing.T) {
2731
db, _, cfg, cleanup := setupTestDB(t)
2832
defer cleanup()

0 commit comments

Comments
 (0)