From d1fac72197e205d9b96a60aee51d9b78d351986e Mon Sep 17 00:00:00 2001 From: tonywangcn Date: Wed, 14 Oct 2026 01:24:16 +0800 Subject: [PATCH 1/2] Measure source admissions and cost with conservative cooldowns --- README.md | 27 +- cmd/freeproxyapi/feed_audit.go | 337 +++++++++++++ cmd/freeproxyapi/feed_audit_test.go | 176 +++++++ cmd/freeproxyapi/main.go | 3 + docs/api.md | 139 +++++- docs/configuration.md | 25 +- docs/redis-schema.md | 55 ++- internal/monitor/config.go | 53 +- internal/monitor/config_test.go | 38 ++ internal/monitor/health.go | 40 +- internal/monitor/metrics.go | 39 +- internal/monitor/runner.go | 96 +++- internal/monitor/source_efficiency.go | 396 +++++++++++++++ .../source_efficiency_integration_test.go | 128 +++++ internal/monitor/source_efficiency_test.go | 261 ++++++++++ internal/monitor/source_learning.go | 80 ++- internal/monitor/source_learning_runner.go | 30 +- internal/monitor/source_learning_test.go | 163 +++++++ internal/monitor/stats_leader.go | 57 ++- internal/source/fetcher.go | 79 ++- internal/source/fetcher_cost_test.go | 151 ++++++ internal/store/redis.go | 88 +++- internal/store/source_efficiency.go | 22 + internal/store/source_efficiency_test.go | 460 ++++++++++++++++++ internal/store/source_learning.go | 198 +++++++- k8s/overlays/live-local/monitor.json | 2 + 26 files changed, 3004 insertions(+), 139 deletions(-) create mode 100644 cmd/freeproxyapi/feed_audit.go create mode 100644 cmd/freeproxyapi/feed_audit_test.go create mode 100644 internal/monitor/source_efficiency.go create mode 100644 internal/monitor/source_efficiency_integration_test.go create mode 100644 internal/monitor/source_efficiency_test.go create mode 100644 internal/source/fetcher_cost_test.go create mode 100644 internal/store/source_efficiency.go create mode 100644 internal/store/source_efficiency_test.go diff --git a/README.md b/README.md index c6fffd4..5bfc3f2 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,28 @@ go run ./cmd/freeproxyapi parse -input ./examples/proxies.txt Use `-default-scheme` when `host:port` records should use a scheme other than `http`. +### Audit published feeds + +`feed-audit` fetches and parses feed URLs without probing listed proxies, using +Redis, or writing files. Supply only feeds you are permitted to retrieve. It +reports source IDs and aggregate counts/costs; it never prints source URLs, +proxy endpoints, feed bodies, or raw fetch errors. For example, replace the +placeholder with a published feed you are authorized to read: + +```sh +printf '[{"url":"https://feeds.example.net/proxies.txt","scheme":"http"}]\n' \ + | go run ./cmd/freeproxyapi feed-audit -input /dev/stdin +``` + +The command accepts up to 128 sources in a 1 MiB JSON input and emits one JSON +record per input source. Defaults are 20 seconds per source, two concurrent +fetches, 32 MiB per body, and 100,000 accepted records per source. Concurrency +is limited to 1–4, timeout to at most 90 seconds, and byte/record limits are +capped. Reaching a record cap marks that source truncated and ineligible; empty +parsed sets are also ineligible. In either case its set hash is omitted. See +[`docs/api.md`](docs/api.md#feed-audit-cli) for the output fields and +measurement limits. + ### Run with Docker Compose The included Compose setup starts FreeProxyAPI and Redis. It mounts @@ -156,7 +178,7 @@ every `/proxies` filter, the response schema, and error codes, is in | `/get?probe_content=1` | Stable `GET`/`HEAD` plain-text contract for explicitly configured target profiles; served with `Cache-Control: no-store`. Other `/get` queries keep the caller-IP echo behavior. | | `/livez`, `/readyz` | Liveness and readiness (`503` when Redis is unavailable). | | `/`, `/dashboard`, `/dashboard/data` | Built-in homepage and operations dashboard (the dashboard needs `prometheus_url`). | -| `/metrics`, `/report` | Operator endpoints: Prometheus metrics and monitor status. Served on `admin_listen_addr` when set. | +| `/metrics`, `/report`, `/report/sources` | Operator endpoints: Prometheus metrics, monitor status, and aggregate source quality/cost. `/report/sources` omits source URLs and endpoint data. Served on `admin_listen_addr` when set. | | `/internal/api/v1/proxies` | Bearer-token internal API, disabled unless `internal_api_token_file` is set. Served on `admin_listen_addr` when set; keep it on an internal network. | ```sh @@ -198,7 +220,8 @@ The settings you will touch first: - `workers` and `global_requests_per_minute` — validation concurrency and the request budget shared by all replicas. - `listen_addr` (default `:8080`) and `admin_listen_addr` — the public listener - and an optional second one for `/metrics`, `/report`, and the internal API. + and an optional second one for `/metrics`, `/report`, `/report/sources`, and + the internal API. `/metrics` exposes feed hostnames and probe settings, so when `admin_listen_addr` is empty those endpoints share the public listener and your ingress must not forward them. diff --git a/cmd/freeproxyapi/feed_audit.go b/cmd/freeproxyapi/feed_audit.go new file mode 100644 index 0000000..3add78f --- /dev/null +++ b/cmd/freeproxyapi/feed_audit.go @@ -0,0 +1,337 @@ +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "net/netip" + "net/url" + "os" + "os/signal" + "sort" + "strconv" + "strings" + "sync" + "syscall" + "time" + + "github.com/Crawlora-org/FreeProxyAPI/internal/endpoint" + "github.com/Crawlora-org/FreeProxyAPI/internal/source" + "github.com/Crawlora-org/FreeProxyAPI/internal/store" +) + +const ( + feedAuditInputMaxBytes = 1 << 20 + feedAuditMaxInputs = 128 + feedAuditMaxBytes = 32 << 20 + feedAuditMaxRecords = 100_000 + feedAuditMaxWorkers = 4 +) + +type feedAuditInput struct { + URL string `json:"url"` + Scheme string `json:"scheme,omitempty"` +} + +type feedAuditResult struct { + SourceID string `json:"source_id"` + InputIndex int `json:"input_index"` + Scheme string `json:"scheme"` + Eligible bool `json:"eligible"` + Truncated bool `json:"truncated"` + ParsedUnique int `json:"parsed_unique"` + Rejected int `json:"rejected"` + ByScheme map[string]int `json:"by_scheme"` + HTTPAttempts int64 `json:"http_attempts"` + NetworkBytes int64 `json:"network_bytes"` + DurationMillis int64 `json:"duration_ms"` + CacheHit bool `json:"cache_hit"` + SetHash string `json:"hash_of_canonical_set,omitempty"` + ErrorCode string `json:"error_code,omitempty"` + HTTPStatus int `json:"http_status,omitempty"` +} + +type feedAuditFetcher interface { + StreamMeasured(context.Context, string, func(io.Reader) error) (source.FetchStats, error) +} + +type feedAuditOptions struct { + Timeout time.Duration + Concurrency int + MaxBytes int64 + MaxRecords int +} + +type feedAuditFlags struct { + inputPath string + options feedAuditOptions +} + +func feedAuditCommand(args []string, out io.Writer) error { + flags := flag.NewFlagSet("feed-audit", flag.ContinueOnError) + flags.SetOutput(io.Discard) + inputPath := flags.String("input", "/dev/stdin", "JSON array of feed URLs") + timeout := flags.Duration("timeout", 20*time.Second, "maximum time per source") + concurrency := flags.Int("concurrency", 2, "maximum concurrent feed requests (1-4)") + maxBytes := flags.Int64("max-bytes", feedAuditMaxBytes, "maximum response bytes per source") + maxRecords := flags.Int("max-records", feedAuditMaxRecords, "maximum accepted records per source") + if err := flags.Parse(args); err != nil { + return err + } + if flags.NArg() != 0 { + return fmt.Errorf("unexpected positional arguments") + } + options := feedAuditOptions{Timeout: *timeout, Concurrency: *concurrency, MaxBytes: *maxBytes, MaxRecords: *maxRecords} + if err := validateFeedAuditOptions(options); err != nil { + return err + } + file, err := os.Open(*inputPath) + if err != nil { + return fmt.Errorf("could not open audit input") + } + defer file.Close() + inputBytes, err := io.ReadAll(io.LimitReader(file, feedAuditInputMaxBytes+1)) + if err != nil || len(inputBytes) > feedAuditInputMaxBytes { + return fmt.Errorf("audit input is unreadable or too large") + } + inputs, err := decodeFeedAuditInputs(inputBytes, feedAuditMaxInputs) + if err != nil { + return err + } + fetcher := source.NewFetcher(options.Timeout, options.MaxBytes) + ctx, stop := signalContext() + defer stop() + results, err := runFeedAudit(ctx, inputs, options, fetcher) + if err != nil { + return err + } + encoder := json.NewEncoder(out) + for _, result := range results { + if err := encoder.Encode(result); err != nil { + return fmt.Errorf("could not write audit result") + } + } + return nil +} + +// signalContext separates the command's cancellation handling from the audit +// runner so its validation and network behavior can be tested hermetically. +func signalContext() (context.Context, context.CancelFunc) { + return signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) +} + +func validateFeedAuditOptions(options feedAuditOptions) error { + if options.Timeout <= 0 || options.Timeout > 90*time.Second { + return fmt.Errorf("-timeout must be positive and no greater than 90s") + } + if options.Concurrency < 1 || options.Concurrency > feedAuditMaxWorkers { + return fmt.Errorf("-concurrency must be between 1 and 4") + } + if options.MaxBytes < 1 || options.MaxBytes > feedAuditMaxBytes { + return fmt.Errorf("-max-bytes must be between 1 and 33554432") + } + if options.MaxRecords < 1 || options.MaxRecords > feedAuditMaxRecords { + return fmt.Errorf("-max-records must be between 1 and 100000") + } + return nil +} + +func decodeFeedAuditInputs(data []byte, maxInputs int) ([]feedAuditInput, error) { + var inputs []feedAuditInput + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&inputs); err != nil || inputs == nil { + return nil, fmt.Errorf("audit input must be a JSON array of {url, scheme} records") + } + var trailing any + if err := decoder.Decode(&trailing); err != io.EOF { + return nil, fmt.Errorf("audit input must contain exactly one JSON array") + } + if len(inputs) > maxInputs { + return nil, fmt.Errorf("audit input has too many source records") + } + for i := range inputs { + inputs[i].URL = strings.TrimSpace(inputs[i].URL) + inputs[i].Scheme = strings.ToLower(strings.TrimSpace(inputs[i].Scheme)) + if inputs[i].Scheme == "" { + inputs[i].Scheme = "http" + } + if inputs[i].Scheme != "http" && inputs[i].Scheme != "socks4" && inputs[i].Scheme != "socks5" { + return nil, fmt.Errorf("audit input source %d has an unsupported parser scheme", i) + } + if err := validateFeedAuditURL(inputs[i].URL); err != nil { + return nil, fmt.Errorf("audit input source %d has an invalid URL", i) + } + } + return inputs, nil +} + +func validateFeedAuditURL(raw string) error { + if raw == "" || len(raw) > 8192 || strings.Contains(raw, "#") { + return errors.New("invalid URL") + } + u, err := url.ParseRequestURI(raw) + if err != nil || u.User != nil || u.Fragment != "" || u.Opaque != "" || u.Hostname() == "" { + return errors.New("invalid URL") + } + if ip, err := netip.ParseAddr(u.Hostname()); err == nil && endpoint.UnsafeIP(ip) { + return errors.New("non-public source address") + } + if strings.TrimSuffix(strings.ToLower(u.Hostname()), ".") == "localhost" { + return errors.New("non-public source hostname") + } + if u.Scheme != "http" && u.Scheme != "https" { + return errors.New("unsupported URL scheme") + } + if port := u.Port(); port != "" { + parsedPort, err := strconv.ParseUint(port, 10, 16) + if err != nil || parsedPort == 0 { + return errors.New("invalid URL port") + } + } + return nil +} + +func runFeedAudit(ctx context.Context, inputs []feedAuditInput, options feedAuditOptions, fetcher feedAuditFetcher) ([]feedAuditResult, error) { + if err := validateFeedAuditOptions(options); err != nil { + return nil, err + } + if len(inputs) > feedAuditMaxInputs { + return nil, fmt.Errorf("audit input has too many source records") + } + // Validate the entire batch before starting any request, including callers + // that use this helper directly rather than decodeFeedAuditInputs. + validated := make([]feedAuditInput, len(inputs)) + copy(validated, inputs) + for i := range validated { + validated[i].URL = strings.TrimSpace(validated[i].URL) + validated[i].Scheme = strings.ToLower(strings.TrimSpace(validated[i].Scheme)) + if validated[i].Scheme == "" { + validated[i].Scheme = "http" + } + if (validated[i].Scheme != "http" && validated[i].Scheme != "socks4" && validated[i].Scheme != "socks5") || validateFeedAuditURL(validated[i].URL) != nil { + return nil, fmt.Errorf("audit input source %d is invalid", i) + } + } + if fetcher == nil { + return nil, fmt.Errorf("audit fetcher is required") + } + results := make([]feedAuditResult, len(validated)) + jobs := make(chan int) + workers := options.Concurrency + if workers > len(validated) { + workers = len(validated) + } + var wg sync.WaitGroup + for worker := 0; worker < workers; worker++ { + wg.Add(1) + go func() { + defer wg.Done() + for index := range jobs { + results[index] = auditOneFeed(ctx, index, validated[index], options, fetcher) + } + }() + } + for i := range validated { + jobs <- i + } + close(jobs) + wg.Wait() + return results, nil +} + +func auditOneFeed(parent context.Context, index int, input feedAuditInput, options feedAuditOptions, fetcher feedAuditFetcher) feedAuditResult { + result := feedAuditResult{ + SourceID: store.StableSourceID(input.URL, input.Scheme), + InputIndex: index, + Scheme: input.Scheme, + ByScheme: make(map[string]int), + } + ctx, cancel := context.WithTimeout(parent, options.Timeout) + defer cancel() + unique := make(map[string]string) + acceptedRecords := 0 + stats, err := fetcher.StreamMeasured(ctx, input.URL, func(r io.Reader) error { + rejected, parseErr := endpoint.VisitFeedProxies(r, input.Scheme, func(proxy endpoint.Proxy) error { + acceptedRecords++ + canonical := proxy.String() + hash := sha256.Sum256([]byte(canonical)) + unique[hex.EncodeToString(hash[:])] = canonical + if acceptedRecords >= options.MaxRecords { + result.Truncated = true + return errFeedAuditRecordLimit + } + return nil + }) + result.Rejected = rejected + if errors.Is(parseErr, errFeedAuditRecordLimit) { + return nil + } + return parseErr + }) + result.HTTPAttempts = stats.HTTPAttempts + result.NetworkBytes = stats.NetworkBytes + result.DurationMillis = stats.Duration.Milliseconds() + result.CacheHit = stats.CacheHit + result.ParsedUnique = len(unique) + for _, canonical := range unique { + colon := strings.Index(canonical, "://") + if colon > 0 { + result.ByScheme[canonical[:colon]]++ + } + } + if err != nil { + result.ErrorCode, result.HTTPStatus = feedAuditErrorCode(err) + return result + } + if result.Truncated { + result.ErrorCode = "record_limit" + return result + } + if result.ParsedUnique == 0 { + result.ErrorCode = "empty_feed" + return result + } + result.Eligible = true + result.SetHash = canonicalSetHash(unique) + return result +} + +var errFeedAuditRecordLimit = errors.New("feed audit record limit reached") + +func feedAuditErrorCode(err error) (string, int) { + if errors.Is(err, source.ErrSourceTooLarge) { + return "byte_limit", 0 + } + if errors.Is(err, context.DeadlineExceeded) { + return "timeout", 0 + } + if errors.Is(err, context.Canceled) { + return "canceled", 0 + } + var statusErr *source.HTTPStatusError + if errors.As(err, &statusErr) { + return "http_status", statusErr.StatusCode + } + return "fetch_or_parse_error", 0 +} + +func canonicalSetHash(unique map[string]string) string { + canonical := make([]string, 0, len(unique)) + for _, value := range unique { + canonical = append(canonical, value) + } + sort.Strings(canonical) + h := sha256.New() + for _, value := range canonical { + _, _ = io.WriteString(h, value) + _, _ = io.WriteString(h, "\n") + } + return hex.EncodeToString(h.Sum(nil)) +} diff --git a/cmd/freeproxyapi/feed_audit_test.go b/cmd/freeproxyapi/feed_audit_test.go new file mode 100644 index 0000000..aca12c4 --- /dev/null +++ b/cmd/freeproxyapi/feed_audit_test.go @@ -0,0 +1,176 @@ +package main + +import ( + "context" + "fmt" + "io" + "strings" + "sync" + "testing" + "time" + + "github.com/Crawlora-org/FreeProxyAPI/internal/source" + "github.com/Crawlora-org/FreeProxyAPI/internal/store" +) + +type auditFixtureFetcher struct { + mu sync.Mutex + payloads map[string]string + errors map[string]error + calls map[string]int +} + +func (f *auditFixtureFetcher) StreamMeasured(_ context.Context, rawURL string, visit func(io.Reader) error) (source.FetchStats, error) { + f.mu.Lock() + f.calls[rawURL]++ + payload := f.payloads[rawURL] + err := f.errors[rawURL] + f.mu.Unlock() + stats := source.FetchStats{HTTPAttempts: 1, NetworkBytes: int64(len(payload)), Duration: 5 * time.Millisecond} + if err != nil { + return stats, err + } + err = visit(strings.NewReader(payload)) + return stats, err +} + +func TestDecodeFeedAuditInputsRejectsUnsafeURLsBeforeFetching(t *testing.T) { + options := testFeedAuditOptions() + fetcher := &auditFixtureFetcher{calls: make(map[string]int)} + inputs := []feedAuditInput{ + {URL: "https://feed.example/list", Scheme: "http"}, + {URL: "https://user:secret@feed.example/list", Scheme: "http"}, + } + if _, err := runFeedAudit(context.Background(), inputs, options, fetcher); err == nil { + t.Fatal("invalid batch was accepted") + } + if len(fetcher.calls) != 0 { + t.Fatalf("invalid batch started requests: %#v", fetcher.calls) + } + + for _, raw := range []string{ + "file:///tmp/feed", + "ftp://feed.example/list", + "https://user:pass@feed.example/list", + "https://feed.example/list#fragment", + "//feed.example/list", + } { + if err := validateFeedAuditURL(raw); err == nil { + t.Errorf("validateFeedAuditURL(%q) succeeded", raw) + } + } + for _, scheme := range []string{"auto", "https"} { + data := []byte(`[{"url":"https://feed.example/list","scheme":"` + scheme + `"}]`) + _, err := decodeFeedAuditInputs(data, 10) + if err == nil { + t.Errorf("decode accepted parser scheme %q", scheme) + } + } + if _, err := decodeFeedAuditInputs([]byte(`[{"url":"https://feed.example/list","scheme":"SOCKS4"}]`), 10); err != nil { + t.Errorf("decode rejected case-insensitive SOCKS4: %v", err) + } + decoded, err := decodeFeedAuditInputs([]byte(`[{"url":"https://feed.example/list"}]`), 10) + if err != nil || len(decoded) != 1 || decoded[0].Scheme != "http" { + t.Fatalf("default parser scheme = %#v, %v", decoded, err) + } +} + +func TestDecodeFeedAuditInputsRejectsMalformedRecords(t *testing.T) { + for _, raw := range []string{ + `{ "url": "https://feed.example/list" }`, + `[{"url":"https://feed.example/list","unexpected":true}]`, + `[{"url":"https://feed.example/list"}] []`, + `null`, + } { + if _, err := decodeFeedAuditInputs([]byte(raw), 10); err == nil { + t.Errorf("decodeFeedAuditInputs accepted %s", raw) + } + } + if _, err := decodeFeedAuditInputs([]byte(`[{"url":"https://one.example/list"},{"url":"https://two.example/list"}]`), 1); err == nil { + t.Fatal("decodeFeedAuditInputs accepted too many source records") + } +} + +func TestRunFeedAuditAggregatesDeduplicatesAndHashesOnlyCompleteSet(t *testing.T) { + const feedURL = "https://feed.example/inventory" + fetcher := &auditFixtureFetcher{ + payloads: map[string]string{feedURL: "proxy.example:8080\nproxy.example:8080\nsocks5://proxy2.example:1080\ninvalid\n"}, + calls: make(map[string]int), + } + results, err := runFeedAudit(context.Background(), []feedAuditInput{{URL: feedURL}}, testFeedAuditOptions(), fetcher) + if err != nil || len(results) != 1 { + t.Fatalf("runFeedAudit = %#v, %v", results, err) + } + result := results[0] + if result.SourceID != store.StableSourceID(feedURL, "http") || result.InputIndex != 0 || result.Scheme != "http" { + t.Fatalf("identity fields = %+v", result) + } + if !result.Eligible || result.Truncated || result.ParsedUnique != 2 || result.Rejected != 1 { + t.Fatalf("feed counts = %+v", result) + } + if result.ByScheme["http"] != 1 || result.ByScheme["socks5"] != 1 || result.HTTPAttempts != 1 || result.NetworkBytes == 0 || result.DurationMillis != 5 || result.CacheHit { + t.Fatalf("feed metrics = %+v", result) + } + if len(result.SetHash) != 64 { + t.Fatalf("canonical-set hash = %q, want 64 hex chars", result.SetHash) + } + if _, err := runFeedAudit(context.Background(), []feedAuditInput{{URL: feedURL}}, testFeedAuditOptions(), fetcher); err != nil { + t.Fatal(err) + } + if results[0].SetHash != result.SetHash { + t.Fatal("canonical-set hash changed for identical contents") + } +} + +func TestRunFeedAuditMarksRecordLimitIneligible(t *testing.T) { + const feedURL = "https://feed.example/limited" + fetcher := &auditFixtureFetcher{ + payloads: map[string]string{feedURL: "proxy1.example:8080\nproxy2.example:8080\nproxy3.example:8080\n"}, + calls: make(map[string]int), + } + options := testFeedAuditOptions() + options.MaxRecords = 2 + results, err := runFeedAudit(context.Background(), []feedAuditInput{{URL: feedURL}}, options, fetcher) + if err != nil || len(results) != 1 { + t.Fatalf("runFeedAudit = %#v, %v", results, err) + } + if results[0].Eligible || !results[0].Truncated || results[0].ErrorCode != "record_limit" || results[0].ParsedUnique != 2 || results[0].SetHash != "" { + t.Fatalf("truncated result = %+v", results[0]) + } +} + +func TestRunFeedAuditUsesRedactedErrorCodes(t *testing.T) { + const feedURL = "https://feed.example/unavailable" + fetcher := &auditFixtureFetcher{ + errors: map[string]error{feedURL: &source.HTTPStatusError{StatusCode: 503, Status: "503 Service Unavailable"}}, + calls: make(map[string]int), + } + results, err := runFeedAudit(context.Background(), []feedAuditInput{{URL: feedURL}}, testFeedAuditOptions(), fetcher) + if err != nil || len(results) != 1 { + t.Fatalf("runFeedAudit = %#v, %v", results, err) + } + if results[0].Eligible || results[0].ErrorCode != "http_status" || results[0].HTTPStatus != 503 { + t.Fatalf("status result = %+v", results[0]) + } + code, status := feedAuditErrorCode(fmt.Errorf("contains private source data")) + if code != "fetch_or_parse_error" || status != 0 { + t.Fatalf("generic error classification = %q/%d", code, status) + } +} + +func testFeedAuditOptions() feedAuditOptions { + return feedAuditOptions{Timeout: 20 * time.Second, Concurrency: 2, MaxBytes: feedAuditMaxBytes, MaxRecords: feedAuditMaxRecords} +} + +func TestFeedAuditRejectsEmptyFeed(t *testing.T) { + result := auditOneFeed(context.Background(), 0, feedAuditInput{URL: "https://feeds.example/empty", Scheme: "http"}, feedAuditOptions{Timeout: time.Second, Concurrency: 1, MaxBytes: 1024, MaxRecords: 10}, emptyAuditFetcher{}) + if result.Eligible || result.ErrorCode != "empty_feed" { + t.Fatalf("empty feed eligible: %+v", result) + } +} + +type emptyAuditFetcher struct{} + +func (emptyAuditFetcher) StreamMeasured(ctx context.Context, u string, visit func(io.Reader) error) (source.FetchStats, error) { + return source.FetchStats{}, visit(strings.NewReader("")) +} diff --git a/cmd/freeproxyapi/main.go b/cmd/freeproxyapi/main.go index 0457a58..49b282b 100644 --- a/cmd/freeproxyapi/main.go +++ b/cmd/freeproxyapi/main.go @@ -26,6 +26,8 @@ func main() { err = parseCommand(os.Args[2:], os.Stdout) case "monitor": err = monitorCommand(os.Args[2:]) + case "feed-audit": + err = feedAuditCommand(os.Args[2:], os.Stdout) case "help", "-h", "--help": usage(os.Stdout) return @@ -110,6 +112,7 @@ func usage(out io.Writer) { Usage: FreeProxyAPI parse -input FILE [-default-scheme http] + FreeProxyAPI feed-audit [-input /dev/stdin] [-timeout 20s] [-concurrency 2] [-max-bytes 33554432] [-max-records 100000] FreeProxyAPI monitor -config FILE [-worker-id ID] `) } diff --git a/docs/api.md b/docs/api.md index 241c13f..a02eb6c 100644 --- a/docs/api.md +++ b/docs/api.md @@ -207,10 +207,143 @@ Nothing is stored. | `/` and `/dashboard` | Built-in homepage and operations dashboard. | | `/dashboard/data?range=1h\|6h\|24h` | Dashboard JSON backed by Prometheus; cached 15 seconds per range. | +## Operator source report + +`GET /report/sources` returns bounded source quality and cost aggregates for the +current configured source catalog. It is an operator endpoint served on the +same `admin_listen_addr` listener as `/report` and `/metrics`; if +`admin_listen_addr` is empty, it shares the main listener for compatibility. +Keep that listener private or prevent public ingress from forwarding it. The +response contains 32-character lowercase hexadecimal source IDs and effective +parser schemes, not source URLs, feed contents, or proxy endpoints. Each ID is +derived from the exact trimmed feed URL and parser-scheme hint. It is `no-store` +and accepts `GET` and `HEAD`. + +When source learning is disabled, the response status is `disabled`. Otherwise +it includes a bounded current-catalog snapshot. `quality_captured_at`, +`quality_age_seconds`, and `quality_stale` describe the local in-process cache +(up to one minute); `source_counts_at`, `source_counts_age_seconds`, and +`source_counts_observed` describe the latest validated-set scan/aggregate. The +quality and candidate-count reads occur separately, so the report is not an +atomic cross-section. Check the timestamps before comparing fields. If source +counts were not observed, the per-source `validated` and `routing_eligible` +fields and the unattributed totals are omitted. If a quality field has no +measurement yet or its quality record is corrupt, its `*_measured` flag is +false and its numeric fields are omitted; an omitted value is unknown, not zero. + +The report distinguishes source history from current candidate attribution: +`admitted` counts actual new candidate-set insertions, not staged source rows. +Each candidate keeps one selected source origin. Duplicate feed appearances +do not create multiple source votes. `unattributed_validated` and +`unattributed_routing_eligible` count validated candidates whose source ID is +missing or no longer belongs to the configured catalog; they cannot be assigned +to a current source. The shared Redis aggregate uses `unattributed_routing` for +the routing-eligible counterpart; the report field uses the more explicit +`unattributed_routing_eligible` name. + +The shared validated aggregate carries source-count maps between replicas, but +the public `/stats` schema deliberately omits source attribution. Operators see +that attribution through `/report/sources` only. + +`routing_eligible` is the subset of validated candidates meeting the current +strict router evidence: success ratio at least 80%, a positive success streak, +a passing fresh HTTPS/CONNECT check, and a fresh clean tamper check, with age +bounded by `classification_max_age`. It is a point-in-time filter count, not a +claim that the proxy will remain usable. + +Admission refresh fields (`admission_refreshes`, `last_admitted`, +`zero_admission_streak`, and `last_admission_at`) are updated only for a whole +successful refresh pass and sources whose pass was not truncated or affected by +the candidate cap. Failed, skipped, truncated, and capped sources do not add a +zero-admission observation. `admitted` itself counts atomic new set insertions +and may include accepted additions from a pass whose later step failed; it is +not the same as a completed admission observation. + +Fetch cost fields sum HTTP client attempts, consumed HTTP response-body bytes, +and elapsed stream work. Attempts count calls to the HTTP client; bytes exclude +headers and TLS framing, and a conditional `304` has zero response-body bytes +even though the cached body is replayed to the parser. Elapsed time covers the +whole stream callback, including parsing and staging, rather than pure HTTP +latency. File feeds contribute zero HTTP attempts and network bytes. + +Probe cost fields count committed probe outcomes only, after lease and probe +contract checks. Elapsed work includes the primary probe and any auxiliary +checks executed within that lease. It is not total CPU time or a complete +source-cost measure. Changing the base probe contract resets probe-cost +history for the new contract; legacy quality hashes without these fields remain +unmeasured until new observations arrive. A stored probe contract alone is not +enough: probe-cost fields are measured only when the current contract matches +and committed probe-completion counters exist. Source quality hashes expire +after 30 days, so cumulative counters restart after expiry. + +Per-source Prometheus efficiency series are published only by the stats leader +when both `source_learning_enabled` and `stats_leader_enabled` are true. With +either feature disabled, these per-source series are absent. Only the unique +lease holder exports them; followers suppress their per-source series. The +JSON report remains usable independently of stats-leader publication when +source learning is enabled. The metric families include +`freeproxyapi_source_admitted_total`, +`freeproxyapi_source_admission_refreshes_total`, +`freeproxyapi_source_http_attempts_total`, +`freeproxyapi_source_response_bytes_total`, +`freeproxyapi_source_fetch_work_milliseconds_total`, +`freeproxyapi_source_probe_completions_total`, +`freeproxyapi_source_probe_successful_completions_total`, +`freeproxyapi_source_probe_work_milliseconds_total`, +`freeproxyapi_source_validated`, +`freeproxyapi_source_routing_eligible`, and +`freeproxyapi_source_zero_admission_streak`. The unattributed count and +snapshot families `freeproxyapi_source_unattributed_validated`, +`freeproxyapi_source_unattributed_routing_eligible`, +`freeproxyapi_source_efficiency_quality_snapshot_age_seconds`, +`freeproxyapi_source_efficiency_counts_snapshot_age_seconds`, and +`freeproxyapi_source_efficiency_counts_observed` have no per-source label. +Per-source series use the bounded `source_id` hash label, never a URL or +endpoint. + +## Feed-audit CLI + +The read-only `feed-audit` CLI rechecks feed retrieval and parsing without +probing listed endpoints or connecting to Redis. It reads an input JSON array +from a file (default `/dev/stdin`): + +```json +[{"url":"https://feeds.example.net/proxies.txt","scheme":"http"}] +``` + +The URL must be HTTP(S), without user information or a fragment, and no longer +than 8,192 characters. The complete batch is validated before requests start; +private IP literals and `localhost` are rejected up front, and resolved +hostnames are checked against the source fetcher's public-address dial policy. +`scheme` is the default parser scheme for bare records and may be `http`, +`socks4`, or `socks5`; it defaults to `http`. There is no automatic scheme +inference. The fetcher rejects redirects. The command writes no files and does +not expose source URLs, response bodies, proxy endpoints, or raw fetch errors. + +The JSON input is limited to 1 MiB and 128 source records. +Defaults are a 20-second per-source timeout, two workers, 32 MiB maximum +response body per source, and 100,000 accepted records per source. Concurrency +is restricted to 1–4, timeout to at most 90 seconds, and the byte and record +caps cannot exceed those defaults. Reaching the record cap marks a result +truncated and ineligible; an empty parsed set is also ineligible, and either +result omits the canonical-set hash. Results +are JSON Lines with source ID, input index, parser scheme, eligibility, +truncation, parsed-unique and rejected counts, counts by endpoint scheme, +fetch-cost fields, optional canonical-set hash, and a bounded error code or +HTTP status; HTTP status errors also include the numeric `http_status` field. +Timeouts, cancellation, byte limits, record limits, and other fetch/parse +errors have bounded codes. Eligibility requires a completed fetch and parse, a +nonempty canonical set, and no record cap. + +The CLI's `http_attempts`, `network_bytes`, and `duration_ms` use the same +measurement definitions as the source report: response-body reader bytes only, +cached replay excluded, and duration includes parsing work. + ## Operator endpoints -`/metrics` (Prometheus), `/report` (monitor status and validation counts, never -proxy URLs) and the internal API `/internal/api/v1/proxies` (bearer token, off -unless `internal_api_token_file` is set) are for operators. Serve them on +`/metrics` (Prometheus), `/report` (monitor status and validation counts), +`/report/sources` (source IDs and aggregate measurements), and the internal API +`/internal/api/v1/proxies` (bearer token, off unless +`internal_api_token_file` is set) are for operators. Serve them on `admin_listen_addr` and keep them off the public internet; see the [README](../README.md#configuration) and [deployment guide](deployment.md). diff --git a/docs/configuration.md b/docs/configuration.md index d00d922..1b1800c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -29,7 +29,7 @@ operate and have approval to use. Read the | `sources` | none (required) | Proxy feeds, such as `file:///data/proxies.txt` or an `https://` URL. An optional `scheme@` prefix sets the default scheme for bare `host:port` lines in that feed, for example `socks5@https://example.com/socks5.txt`. | | `default_proxy_scheme` | `http` | Scheme for bare `host:port` records whose feed has no `scheme@` prefix. | | `listen_addr` | `:8080` | Public HTTP listener. | -| `admin_listen_addr` | empty | Optional second listener, for example `127.0.0.1:9090`, for `/metrics`, `/report` and `/internal/api/v1/proxies`. When set, those endpoints are absent from `listen_addr`. Must differ from `listen_addr`. | +| `admin_listen_addr` | empty | Optional second listener, for example `127.0.0.1:9090`, for `/metrics`, `/report`, `/report/sources`, and `/internal/api/v1/proxies`. When set, those endpoints are absent from `listen_addr`. Must differ from `listen_addr`. | | `public_base_url` | empty | Public origin of your deployment, such as `https://proxies.example.com`. Used for canonical, Open Graph and code-sample URLs on the built-in pages. When empty, samples show the origin the visitor reached and canonical tags are omitted. | | `trusted_proxy_cidrs` | none | CIDRs of ingress that strips caller-supplied forwarding headers and writes trusted ones. Leave empty to identify clients by socket peer. | | `internal_api_token_file` | empty | Bearer-token file for the internal API. The API is disabled when this is empty. See [deployment.md](deployment.md#internal-query-api). | @@ -37,10 +37,11 @@ operate and have approval to use. Read the | `public_max_limit` | `1000` (1–1000) | Largest page `/proxies` serves; a larger `limit` is clamped to it. An omitted `limit` is 100, or this value if smaller. Lower it to 100 to cap pages, after `freeproxyapi_public_large_page_requests_total` has stopped growing (see [deployment.md](deployment.md#lowering-the-public-page-size)). | `/metrics` exposes feed hostnames and probe settings. When `admin_listen_addr` -is empty (the compatibility default), `/metrics`, `/report` and the internal API -share the public listener, so your ingress must not forward them. The Compose -example binds the admin listener to the container's loopback; bind -`0.0.0.0:9090` and publish the port only on a network you control to scrape it. +is empty (the compatibility default), `/metrics`, `/report`, `/report/sources` +and the internal API share the public listener, so your ingress must not forward +them. The Compose example binds the admin listener to the container's loopback; +to scrape it, bind `0.0.0.0:9090` and publish the port only on a network you +control. ## Feed intake @@ -52,6 +53,8 @@ example binds the admin listener to the container's loopback; bind | `max_records_per_source` | `250000` | 1–5,000,000 | Most endpoints one feed can add per refresh; the rest are ignored. | | `max_candidates` | `0` (off) | ≥ 0 | When positive, stops new candidates being added once the set reaches this size. | | `source_learning_enabled` | `false` | | Opt in to per-feed ranking from sampled refresh and probe outcomes. Supports at most 4,096 configured feeds; see [Source learning](#source-learning). | +| `source_learning_zero_admission_cooldown` | `0s` (off) | 0–24h | Optional extra cooldown for lower-priority feeds after repeated complete refreshes produce no newly admitted candidates. Protected priority feeds and feeds with a positive observed probe or sufficiently high learned yield are not cooled. | +| `source_learning_zero_admission_refreshes` | `3` | 3–100 | Consecutive complete, uncapped and untruncated zero-admission refreshes required before the optional cooldown can apply. | HTTP feeds that return `ETag` or `Last-Modified` can be fetched conditionally. A `304 Not Modified` reuses the previously downloaded body and still parses @@ -79,11 +82,23 @@ exploration. Failed fetches use a separate capped retry cooldown. Learning changes source refresh order; it does not change proxy endpoint validation or the listing threshold. +The optional `source_learning_zero_admission_cooldown` adds a separate guard +against repeatedly refreshing low-priority sources that add no new candidate +IDs. It defaults to disabled (`0s`) and is bounded to 24 hours. The configured +`source_learning_zero_admission_refreshes` threshold defaults to 3 and must be +3–100. Only complete successful, uncapped and untruncated refresh passes count +toward this guard; failed, skipped, truncated, or candidate-cap-affected passes +are excluded. Sources at fallback priority 10 or better, sources with a positive +observed probe outcome, and sources whose learned probe yield clears the +configured minimum are protected. The one-in-ten exploration allowance remains +in effect for cooled feeds. + The setting is off by default and has no environment override. Quality hashes expire after 30 days, and only the active configured catalog is attributed. `/report` exposes `source_learning_local`; its `prepared` flag and generation distinguish an actual local refresh plan from an instance that has not planned a refresh. The local plan is not a cluster-wide source report. See +[the operator source report](api.md#operator-source-report) and [Redis schema](redis-schema.md#source-learning-and-quality-history). ## Validation diff --git a/docs/redis-schema.md b/docs/redis-schema.md index e257e24..b55b22c 100644 --- a/docs/redis-schema.md +++ b/docs/redis-schema.md @@ -11,7 +11,7 @@ prefix. Every key below is relative to `namespace`, which defaults to | `:candidates` | SET | SHA-256 hex of every canonical endpoint URL currently tracked. | | `:source-unique` | STRING | Cross-source unique canonical endpoint count from the latest completed source refresh. | | `:source-refresh-stats` | HASH | `parsed` valid records and `succeeded` source count from the latest completed source refresh. | -| `:proxy:` | HASH | Per-candidate record. `url` is the credential-free endpoint string; `last_checked_at_ms`, `last_ok_at_ms` (time of the latest successful probe), `last_status` (`ok`/`failed`), `last_http_status`, `last_latency_ms`, `last_error` (a short failure code from `probe.ClassifyError`, at most 32 bytes, such as `connect_timeout`, `refused`, or `bad_status_4xx`; empty after a success — raw transport errors are not stored, which keeps every field under `hash-max-listpack-value` so the hash stays a compact listpack), the `consecutive_failures` streak, and stability history — `results10` (last ten outcomes as 1/0), `ok_ratio_pct`, `latency_ewma_ms`, `jitter_ewma_ms` — describe history. Optional `country` (ISO), `asn`, `anonymity` (elite/anonymous/transparent), `exit_ip`, and `exit_country` (GeoIP country of the observed exit IP) annotations come from an operator-supplied GeoIP database plus an optional configured echo endpoint, and are never cleared by a later empty lookup. `anonymity_checked_at_ms` is set whenever any of `anonymity`/`exit_ip`/`exit_country` is written; readers treat those fields as unknown once older than `classification_max_age` or when the timestamp is absent. A sampled HTTPS-capability check writes `https_ok` (`1`/`0`) with `https_checked_at_ms`, and `https_tunnel_scheme` (`http`, `https`, or a SOCKS scheme) after a successful check; these are unchanged when no check ran. An anonymity echo check that returned 200 and was compared for tampering writes `tampered` (`1` when the proxy modified the echo body or injected response headers, `0` when clean) with `tamper_checked_at_ms`; both are unchanged when no tamper check ran, and readers treat `tampered` as unknown once older than `classification_max_age` or when the timestamp is absent. Lease fields are `lease_token`, `lease_owner`, and `lease_expires_at_ms`. | +| `:proxy:` | HASH | Per-candidate record. `url` is the credential-free endpoint string; when source learning is enabled, `source_id` is the selected first source origin while that source remains configured. `last_checked_at_ms`, `last_ok_at_ms` (time of the latest successful probe), `last_status` (`ok`/`failed`), `last_http_status`, `last_latency_ms`, `last_error` (a short failure code from `probe.ClassifyError`, at most 32 bytes, such as `connect_timeout`, `refused`, or `bad_status_4xx`; empty after a success — raw transport errors are not stored, which keeps every field under `hash-max-listpack-value` so the hash stays a compact listpack), the `consecutive_failures` streak, and stability history — `results10` (last ten outcomes as 1/0), `ok_ratio_pct`, `latency_ewma_ms`, `jitter_ewma_ms` — describe history. Optional `country` (ISO), `asn`, `anonymity` (elite/anonymous/transparent), `exit_ip`, and `exit_country` (GeoIP country of the observed exit IP) annotations come from an operator-supplied GeoIP database plus an optional configured echo endpoint, and are never cleared by a later empty lookup. `anonymity_checked_at_ms` is set whenever any of `anonymity`/`exit_ip`/`exit_country` is written; readers treat those fields as unknown once older than `classification_max_age` or when the timestamp is absent. A sampled HTTPS-capability check writes `https_ok` (`1`/`0`) with `https_checked_at_ms`, and `https_tunnel_scheme` (`http`, `https`, or a SOCKS scheme) after a successful check; these are unchanged when no check ran. An anonymity echo check that returned 200 and was compared for tampering writes `tampered` (`1` when the proxy modified the echo body or injected response headers, `0` when clean) with `tamper_checked_at_ms`; both are unchanged when no tamper check ran, and readers treat `tampered` as unknown once older than `classification_max_age` or when the timestamp is absent. Lease fields are `lease_token`, `lease_owner`, and `lease_expires_at_ms`. | | `:tomb:` | STRING | Eviction tombstone `":"` written when a candidate is evicted after `max_consecutive_failures` or discarded by `discard_failed_candidates`. Each strike blocks re-adding for `evicted_backoff_base * 2^(strikes-1)` capped at `evicted_backoff_max`; source refreshes skip (no SADD/HSET/ZADD) endpoints that are not current candidates while blocked, and never priority-promote them. The key expires `block + max(block, 2 * max(1, max_consecutive_failures) * failed_retest_interval)` after the eviction, so strike memory survives one re-add and re-eviction cycle and then disappears on its own. | | `:pending` | ZSET | Candidates waiting to be tested. Score is due-at epoch milliseconds. | | `:leased` | ZSET | Candidates currently leased across all replicas. Score is lease-expiry epoch milliseconds. | @@ -25,7 +25,7 @@ prefix. Every key below is relative to `namespace`, which defaults to | `:once::lock` | STRING | Token-owned lock held while a one-shot job runs. Renewed every TTL/3 by the owner; if a renewal finds another token, the job's context is cancelled and no completion marker is written. | | `:probe-budget:` | STRING counter | Global per-minute probe budget shared by every replica. Incremented and given a two-minute expiry atomically. Replicas reserve permits in chunks (`probe_permit_chunk`, default `max(10, workers/4)`) with a Lua `INCRBY` capped at `limit - used`, so the counter never exceeds the limit; permits a replica still holds when the minute ends are discarded, bounding waste to one chunk per replica per window. | | `:stats-leader` | STRING | Random token of the replica that holds the stats-leader lease. Taken with `SET NX PX` semantics and renewed on every stats publish (TTL `stats_leader_ttl`, default 90s); released on graceful shutdown. Only the holder scans the validated set. | -| `:validated-aggregate` | STRING (JSON) | Identity-free validated-set aggregate written by the stats leader, only while its lease token matches: `computed_at_ms`, `by_country`, `stable_by_country`, `by_asn`, `by_anonymity`, `by_latency_band`, `stable`, `band_known`, `geo_mismatch`. TTL `stats_aggregate_ttl` (default 5m). Followers ignore it once older than two publish intervals (60s) and scan locally instead. | +| `:validated-aggregate` | STRING (JSON) | Validated-set aggregate written by the stats leader, only while its lease token matches: `computed_at_ms`, `by_country`, `stable_by_country`, `by_asn`, `by_anonymity`, `by_latency_band`, `stable`, `band_known`, `geo_mismatch`, and (when source learning is active) `by_source`, `routing_by_source`, `source_counts_observed`, `unattributed_validated`, and `unattributed_routing`. Source maps contain counts only, keyed by source ID. TTL `stats_aggregate_ttl` (default 5m). Followers ignore it once older than two publish intervals (60s) and scan locally instead. | ## Source learning and quality history @@ -46,6 +46,53 @@ after 30 days. Attribution is limited to the active source catalog; the `source_learning_local` field in `/report` describes this monitor's local plan, not a cluster-wide plan. +The same hash may also hold bounded measurement counters: + +- `admitted` accumulates candidate IDs newly inserted into the tracked candidate + set by atomic upsert, not rows staged by a feed. It can include real insertions + from a refresh that later fails. +- `admission_refreshes`, `last_admitted`, `zero_admission_streak`, and + `last_admission_ms` describe completed whole-refresh observations. They are + updated only when a source participated in a successful, uncapped and + untruncated refresh pass. Failed, skipped, truncated, or candidate-cap-affected + source passes are excluded. `zero_admission_streak` counts consecutive + qualifying observations with zero new candidate IDs. +- `http_attempts`, `network_bytes`, `fetch_duration_ms`, and + `fetch_cost_observations` accumulate source-fetch measurements. Attempts + count HTTP client calls; bytes count consumed HTTP response-body reader bytes, + excluding headers and TLS framing. A `304` cached-body replay has zero + response-body bytes. Duration is elapsed time for the full stream and its + parser/staging callback, not pure HTTP latency. File feeds have zero HTTP + attempts and network bytes. +- `probe_completions`, `probe_successful_completions`, and `probe_duration_ms` + accumulate primary and auxiliary probe work only when the result commits under + a valid lease and current base-probe contract. Contract changes reset these + counters alongside sampled probe quality for the new contract. Duration is + elapsed work, not CPU time or an all-inclusive source cost. A stored + `probe_contract` by itself does not mean probe-cost fields were measured; + committed completion counters must exist and the contract must match the + current configuration. + +These fields are additive and omitted from report output until measured; +legacy hashes do not imply zero observations. The source-quality hash expires +after 30 days, so its cumulative counters reset after expiry. `/report/sources` +reads a bounded quality snapshot and the validated-source count snapshot +separately. `quality_captured_at`/`quality_age_seconds` and +`source_counts_at`/`source_counts_age_seconds` expose their freshness; the two +reads are not atomic. Missing or retired candidate provenance contributes to +`unattributed_validated` and `unattributed_routing`, rather than being assigned +to a configured source. +The aggregate JSON field `unattributed_routing` is exposed by the operator +report as `unattributed_routing_eligible`; neither field is part of public +`/stats`. + +Per-source validated counts in `validated-aggregate` are only an aggregate +snapshot. `routing_by_source` uses current strict routing evidence: at least 80% +success ratio, positive success streak, fresh passing HTTPS/CONNECT evidence, +and fresh clean tamper evidence within `classification_max_age`. Source IDs +are hashes of the exact trimmed source URL and parser-scheme hint; the URL is +not stored in the source-ID label or report. + Candidate hashes may additionally hold `success_streak`, optional exit fields, `target_profiles`, and `reliability_started_at_ms` plus `reliability_buckets`. The reliability value is a sparse comma-separated sequence of @@ -65,8 +112,8 @@ replace it while that origin remains configured. When an origin is removed, a later accepted upsert may attribute the candidate to a currently active source. Each source-quality hash also stores a `probe_contract` fingerprint for the base target URL, probe mode, and expected body. Changed contracts reset sampled -probe counters before accumulating new evidence; fetch counters and reliability -history remain separate. +probe counters and committed probe-cost counters before accumulating new +evidence; fetch counters and proxy reliability history remain separate. Each source-quality hash contains fetch counters, incremental unique/duplicate counts, cooldown timestamps, and decayed probe sample/pass diff --git a/internal/monitor/config.go b/internal/monitor/config.go index cc61551..174ec03 100644 --- a/internal/monitor/config.go +++ b/internal/monitor/config.go @@ -37,11 +37,12 @@ const ( defaultListenAddr = ":8080" defaultProbeTargetMode = "standard" - defaultProbeConnectTimeout = 3 * time.Second - defaultRetestJitterPct = 10 - maxRetestJitterPct = 50 - defaultEvictedBackoffBase = 2 * time.Hour - defaultEvictedBackoffMax = 168 * time.Hour + defaultProbeConnectTimeout = 3 * time.Second + maxSourceLearningZeroAdmissionCooldown = 24 * time.Hour + defaultRetestJitterPct = 10 + maxRetestJitterPct = 50 + defaultEvictedBackoffBase = 2 * time.Hour + defaultEvictedBackoffMax = 168 * time.Hour // defaultListedFailureRetest re-probes a failed proxy that is still listed // quickly, so a dead proxy leaves /proxies within minutes, not hours. @@ -57,10 +58,12 @@ const ( ) type Config struct { - sourceProbeContract string - SourceLearningEnabled bool - TargetProfiles []TargetProfile - TargetProbeEvery int + sourceProbeContract string + SourceLearningEnabled bool + SourceLearningZeroAdmissionCooldown time.Duration + SourceLearningZeroAdmissionRefreshes int + TargetProfiles []TargetProfile + TargetProbeEvery int RedisURL string Namespace string @@ -239,9 +242,11 @@ func parseSourceSpec(raw string) (SourceSpec, error) { } type fileConfig struct { - SourceLearningEnabled bool `json:"source_learning_enabled"` - TargetProfiles []TargetProfile `json:"target_profiles"` - TargetProbeEvery int `json:"target_probe_every"` + SourceLearningEnabled bool `json:"source_learning_enabled"` + SourceLearningZeroAdmissionCooldown string `json:"source_learning_zero_admission_cooldown"` + SourceLearningZeroAdmissionRefreshes *int `json:"source_learning_zero_admission_refreshes"` + TargetProfiles []TargetProfile `json:"target_profiles"` + TargetProbeEvery int `json:"target_probe_every"` RedisURL string `json:"redis_url"` Namespace string `json:"namespace"` @@ -415,6 +420,13 @@ func LoadConfig(path string) (Config, error) { if config.SourceLearningEnabled && len(config.Sources) > 4096 { return Config{}, fmt.Errorf("source_learning_enabled supports at most 4096 sources") } + config.SourceLearningZeroAdmissionRefreshes = 3 + if file.SourceLearningZeroAdmissionRefreshes != nil { + config.SourceLearningZeroAdmissionRefreshes = *file.SourceLearningZeroAdmissionRefreshes + } + if config.SourceLearningZeroAdmissionRefreshes < 3 || config.SourceLearningZeroAdmissionRefreshes > 100 { + return Config{}, fmt.Errorf("source_learning_zero_admission_refreshes must be between 3 and 100") + } config.SourceSpecs = make([]SourceSpec, 0, len(config.Sources)) for _, raw := range config.Sources { spec, err := parseSourceSpec(raw) @@ -426,6 +438,12 @@ func LoadConfig(path string) (Config, error) { if config.DefaultProxyScheme == "" { config.DefaultProxyScheme = "http" } + if config.SourceLearningZeroAdmissionCooldown, err = nonnegativeDurationOrDefault(file.SourceLearningZeroAdmissionCooldown, 0); err != nil { + return Config{}, fmt.Errorf("source_learning_zero_admission_cooldown: %w", err) + } + if config.SourceLearningZeroAdmissionCooldown > maxSourceLearningZeroAdmissionCooldown { + return Config{}, fmt.Errorf("source_learning_zero_admission_cooldown must not exceed 24h") + } if config.ProbeTargetMode == "" { config.ProbeTargetMode = defaultProbeTargetMode } @@ -660,6 +678,17 @@ func durationOrDefault(raw string, fallback time.Duration) (time.Duration, error return duration, nil } +func nonnegativeDurationOrDefault(raw string, fallback time.Duration) (time.Duration, error) { + if strings.TrimSpace(raw) == "" { + return fallback, nil + } + duration, err := time.ParseDuration(raw) + if err != nil || duration < 0 { + return 0, fmt.Errorf("must be a non-negative Go duration") + } + return duration, nil +} + // RetestPolicy returns the tiered rescheduling policy for probe outcomes. func (c Config) RetestPolicy() store.RetestPolicy { policy := store.RetestPolicy{ diff --git a/internal/monitor/config_test.go b/internal/monitor/config_test.go index 5c3e476..075d43a 100644 --- a/internal/monitor/config_test.go +++ b/internal/monitor/config_test.go @@ -47,6 +47,9 @@ func TestLoadConfigAppliesSafetyDefaults(t *testing.T) { if config.NetworkValidationEnabled { t.Error("network validation must default to disabled") } + if config.SourceLearningZeroAdmissionCooldown != 0 || config.SourceLearningZeroAdmissionRefreshes != 3 { + t.Fatalf("zero-admission defaults = %s/%d, want disabled cooldown and 3 refreshes", config.SourceLearningZeroAdmissionCooldown, config.SourceLearningZeroAdmissionRefreshes) + } if config.DiscardFailedCandidates { t.Error("discard_failed_candidates must default to false") } @@ -173,6 +176,41 @@ func TestLoadConfigEnforcesLimits(t *testing.T) { } } +func TestLoadConfigParsesZeroAdmissionCooldown(t *testing.T) { + config, err := LoadConfig(writeConfig(t, `{ + "sources": ["file:///data/proxies.txt"], + "source_learning_zero_admission_cooldown": "1h", + "source_learning_zero_admission_refreshes": 6 + }`)) + if err != nil { + t.Fatalf("LoadConfig: %v", err) + } + if config.SourceLearningZeroAdmissionCooldown != time.Hour || config.SourceLearningZeroAdmissionRefreshes != 6 { + t.Fatalf("zero-admission config = %s/%d", config.SourceLearningZeroAdmissionCooldown, config.SourceLearningZeroAdmissionRefreshes) + } + if zero, err := LoadConfig(writeConfig(t, `{ + "sources": ["file:///data/proxies.txt"], + "source_learning_zero_admission_cooldown": "0s" + }`)); err != nil || zero.SourceLearningZeroAdmissionCooldown != 0 || zero.SourceLearningZeroAdmissionRefreshes != 3 { + t.Fatalf("explicit disabled cooldown = %s/%d err=%v", zero.SourceLearningZeroAdmissionCooldown, zero.SourceLearningZeroAdmissionRefreshes, err) + } +} + +func TestLoadConfigRejectsInvalidZeroAdmissionCooldown(t *testing.T) { + for _, body := range []string{ + `{"sources":["file:///d/p.txt"],"source_learning_zero_admission_cooldown":"-1s"}`, + `{"sources":["file:///d/p.txt"],"source_learning_zero_admission_cooldown":"25h"}`, + `{"sources":["file:///d/p.txt"],"source_learning_zero_admission_cooldown":"soon"}`, + `{"sources":["file:///d/p.txt"],"source_learning_zero_admission_refreshes":0}`, + `{"sources":["file:///d/p.txt"],"source_learning_zero_admission_refreshes":2}`, + `{"sources":["file:///d/p.txt"],"source_learning_zero_admission_refreshes":101}`, + } { + if _, err := LoadConfig(writeConfig(t, body)); err == nil { + t.Errorf("LoadConfig accepted invalid zero-admission config %s", body) + } + } +} + func TestLoadConfigAcceptsBurstProfileAndDiscardFailedCandidates(t *testing.T) { config, err := LoadConfig(writeConfig(t, `{ "sources": ["file:///d/p.txt"], diff --git a/internal/monitor/health.go b/internal/monitor/health.go index 9ae237b..65aebe9 100644 --- a/internal/monitor/health.go +++ b/internal/monitor/health.go @@ -109,14 +109,20 @@ type refreshReport struct { } type validatedSlices struct { - ByCountry map[string]int64 `json:"by_country,omitempty"` - StableByCountry map[string]int64 `json:"stable_by_country,omitempty"` - ByASN map[string]int64 `json:"by_asn,omitempty"` - ByAnonymity map[string]int64 `json:"by_anonymity,omitempty"` - ByLatencyBand map[string]int64 `json:"by_latency_band,omitempty"` - Stable int64 `json:"stable"` - BandKnown int64 `json:"-"` - GeoMismatch int64 `json:"geo_mismatch"` + UnattributedValidated int64 `json:"-"` + UnattributedRouting int64 `json:"-"` + BySource map[string]int64 `json:"-"` + RoutingBySource map[string]int64 `json:"-"` + SourceCountsObserved bool `json:"-"` + SourceCountsAtMS int64 `json:"-"` + ByCountry map[string]int64 `json:"by_country,omitempty"` + StableByCountry map[string]int64 `json:"stable_by_country,omitempty"` + ByASN map[string]int64 `json:"by_asn,omitempty"` + ByAnonymity map[string]int64 `json:"by_anonymity,omitempty"` + ByLatencyBand map[string]int64 `json:"by_latency_band,omitempty"` + Stable int64 `json:"stable"` + BandKnown int64 `json:"-"` + GeoMismatch int64 `json:"geo_mismatch"` } type reportPayload struct { @@ -454,6 +460,7 @@ func startHealthServer(addr string, runner *Runner) (*healthServer, error) { // listener for compatibility, so the ingress must not forward them. registerAdmin := func(m *http.ServeMux) { m.HandleFunc("/report", h.report) + m.HandleFunc("/report/sources", h.sourceReport) m.Handle("/metrics", withAccuracyMetrics(runner.metrics.Handler(), &runner.accuracy.metrics)) registerInternalAPI(m, runner, loadAPIToken(runner.config.InternalAPITokenFile)) } @@ -1213,3 +1220,20 @@ func parseBool(raw string) bool { return false } } + +// sourceReport returns bounded aggregate-only source attribution on the operator listener. +func (h *healthHandlers) sourceReport(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet && r.Method != http.MethodHead { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + ctx, cancel := context.WithTimeout(r.Context(), readyzTimeout) + defer cancel() + report := h.runner.sourceEfficiencyReport(ctx) + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "no-store") + if r.Method == http.MethodHead { + return + } + _ = json.NewEncoder(w).Encode(report) +} diff --git a/internal/monitor/metrics.go b/internal/monitor/metrics.go index b42a442..50dce5a 100644 --- a/internal/monitor/metrics.go +++ b/internal/monitor/metrics.go @@ -109,8 +109,10 @@ type Metrics struct { geoipGauge atomic.Int64 geoMismatch atomic.Int64 - dynamicMu sync.Mutex - dynamic map[string]int64 + sourceMu sync.Mutex + sourceGauges map[string]int64 + dynamicMu sync.Mutex + dynamic map[string]int64 targetMu sync.Mutex targetMetrics map[string]*targetProbeMetrics @@ -592,6 +594,7 @@ func (m *Metrics) Handler() http.Handler { } fmt.Fprintf(&b, "%s %d\n", s.name, s.value()) } + m.appendSourceEfficiencyMetrics(&b) m.appendTargetProbeMetrics(&b) m.appendProbeOutcomeMetrics(&b) m.appendSourceFailureMetrics(&b) @@ -892,3 +895,35 @@ func (m *Metrics) appendProbeOutcomeMetrics(b *strings.Builder) { } } } + +// ReplaceSourceGauges atomically swaps leader-only, bounded source snapshots. +func (m *Metrics) ReplaceSourceGauges(values map[string]int64) { + copyValues := make(map[string]int64, len(values)) + for k, v := range values { + copyValues[k] = v + } + m.sourceMu.Lock() + m.sourceGauges = copyValues + m.sourceMu.Unlock() +} +func (m *Metrics) appendSourceEfficiencyMetrics(b *strings.Builder) { + m.sourceMu.Lock() + defer m.sourceMu.Unlock() + keys := make([]string, 0, len(m.sourceGauges)) + for k := range m.sourceGauges { + keys = append(keys, k) + } + sort.Strings(keys) + emitted := make(map[string]bool) + for _, k := range keys { + family := k + if idx := strings.IndexByte(k, '{'); idx >= 0 { + family = k[:idx] + } + if !emitted[family] { + fmt.Fprintf(b, "# HELP %s %s\n# TYPE %s %s\n", family, sourceEfficiencyMetricHelp[family], family, sourceEfficiencyMetricType[family]) + emitted[family] = true + } + fmt.Fprintf(b, "%s %d\n", k, m.sourceGauges[k]) + } +} diff --git a/internal/monitor/runner.go b/internal/monitor/runner.go index 1697a64..6485be2 100644 --- a/internal/monitor/runner.go +++ b/internal/monitor/runner.go @@ -92,8 +92,9 @@ type Runner struct { // statsLeader is the shared-aggregate lease (stats_leader.go); permits is // the local global-budget bucket (budget.go). Both are zero-value ready. - statsLeader statsLeaderState - permits permitBucket + statsLeader statsLeaderState + sourceEfficiency sourceEfficiencyState + permits permitBucket } // wakeNotifier is a broadcast edge-triggered notification. Replacing the @@ -397,6 +398,7 @@ func (r *Runner) publishStats(ctx context.Context) { r.metrics.SetPendingSchedule(stats.PendingDue, stats.NextDueAt, stats.HasNextDue, now) slices := r.resolveValidatedSlices(publishCtx, now) + r.publishSourceEfficiency(publishCtx, slices, r.nowFunc()) // Build the complete labeled set first and swap it in atomically so a // concurrent scrape never observes a partially refilled exposition. labeled := make(map[string]int64, len(slices.ByCountry)+len(slices.ByLatencyBand)+len(slices.ByASN)+len(slices.ByAnonymity)) @@ -490,6 +492,8 @@ func (r *Runner) aggregateValidatedSlicesFrom(ctx context.Context, redisStore *s ByASN: map[string]int64{}, ByAnonymity: map[string]int64{}, ByLatencyBand: map[string]int64{}, + BySource: map[string]int64{}, + RoutingBySource: map[string]int64{}, } details, err := redisStore.ValidatedDetails(ctx) if err != nil { @@ -502,7 +506,28 @@ func (r *Runner) aggregateValidatedSlicesFrom(ctx context.Context, redisStore *s // consumers. return r.validatedSlicesSnapshot() } + now := time.Now() + if r.nowFunc != nil { + now = r.nowFunc() + } + out.SourceCountsObserved = true + out.SourceCountsAtMS = now.UnixMilli() + known := make(map[string]bool) + for _, spec := range r.config.SourceSpecs { + known[r.sourceID(spec)] = true + } for _, d := range details { + if known[d.SourceID] { + out.BySource[d.SourceID]++ + if d.RoutingEligible(now, r.config.ClassificationMaxAge) { + out.RoutingBySource[d.SourceID]++ + } + } else { + out.UnattributedValidated++ + if d.RoutingEligible(now, r.config.ClassificationMaxAge) { + out.UnattributedRouting++ + } + } if d.Country != "" && d.ExitCountry != "" && d.Country != d.ExitCountry { out.GeoMismatch++ } @@ -686,6 +711,8 @@ func (r *Runner) startSourceRefreshWorkers(ctx context.Context, stageKey string) // drainSourceRefresh collects per-source results, drains the shared stage // into the validation queue, and records the refresh stats. func (r *Runner) drainSourceRefresh(ctx context.Context, stageKey string, results <-chan sourceRefreshResult) error { + admissions := make(map[string]int64) + admissionExcluded := make(map[string]bool) var ( totalRejected int64 totalParsed int64 @@ -711,6 +738,14 @@ func (r *Runner) drainSourceRefresh(ctx context.Context, stageKey string, result continue } okSources++ + if r.config.SourceLearningEnabled { + id := r.sourceID(result.spec) + if result.truncated { + admissionExcluded[id] = true + } else { + admissions[id] = 0 + } + } totalParsed += int64(result.parsed) totalRejected += int64(result.rejected) } @@ -748,6 +783,12 @@ func (r *Runner) drainSourceRefresh(ctx context.Context, stageKey string, result queueBase := r.nowFunc() err = r.store.DrainSourceStagePrioritized(ctx, stageKey, func(priority int64, batch []string) error { if room == 0 { + if r.config.SourceLearningEnabled { + _, index, decodeErr := store.DecodeSourceStageScore(priority) + if decodeErr == nil && index >= 0 && index < len(r.config.SourceSpecs) { + admissionExcluded[r.sourceID(r.config.SourceSpecs[index])] = true + } + } totalCapped += len(batch) return nil } @@ -756,6 +797,13 @@ func (r *Runner) drainSourceRefresh(ctx context.Context, stageKey string, result return err } totalAdded += counts.Added + if r.config.SourceLearningEnabled { + _, index, err := store.DecodeSourceStageScore(priority) + if err != nil || index < 0 || index >= len(r.config.SourceSpecs) { + return fmt.Errorf("invalid source admission score") + } + admissions[r.sourceID(r.config.SourceSpecs[index])] += int64(counts.Added) + } if room > 0 { if room -= int64(counts.Added); room < 0 { room = 0 @@ -773,6 +821,25 @@ func (r *Runner) drainSourceRefresh(ctx context.Context, stageKey string, result r.recordRefreshError() return err } + if r.config.SourceLearningEnabled { + for id := range admissionExcluded { + delete(admissions, id) + } + ids := make([]string, 0, len(admissions)) + for id := range admissions { + ids = append(ids, id) + } + sort.Strings(ids) + for start := 0; start < len(ids); start += store.MaxSourceQualityReadBatch { + batch := make(map[string]int64) + for _, id := range ids[start:min(start+store.MaxSourceQualityReadBatch, len(ids))] { + batch[id] = admissions[id] + } + if err := r.store.RecordSourceAdmissionRefresh(ctx, batch, r.nowFunc()); err != nil { + log.Printf("source admission observation failed count=%d", len(batch)) + } + } + } if totalCapped > 0 { r.metrics.AddCandidatesCapped(int64(totalCapped)) log.Printf("candidate cap reached max_candidates=%d: skipped %d staged endpoints", r.config.MaxCandidates, totalCapped) @@ -874,11 +941,12 @@ func sourceQueueTime(now time.Time, priority int64) time.Time { } type sourceRefreshResult struct { - spec SourceSpec - stage string - parsed int - rejected int - err error + spec SourceSpec + stage string + parsed int + rejected int + err error + truncated bool } const ( @@ -946,7 +1014,7 @@ func (r *Runner) refreshSourceAttempt(ctx context.Context, spec SourceSpec, stag _ = r.store.DeleteSourceStage(cleanupCtx, sourceKey) }() truncated := false - err = r.fetch.Stream(ctx, spec.URL, func(reader io.Reader) error { + fetchStats, fetchErr := r.fetch.StreamMeasured(ctx, spec.URL, func(reader io.Reader) error { batch := make([]string, 0, sourceStageBatch) flush := func() error { if len(batch) == 0 { @@ -980,6 +1048,16 @@ func (r *Runner) refreshSourceAttempt(ctx context.Context, spec SourceSpec, stag } return flush() }) + err = fetchErr + result.truncated = truncated + if r.config.SourceLearningEnabled { + costCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), releaseTimeout) + costErr := r.store.RecordSourceFetchCost(costCtx, r.sourceID(spec), fetchStats.HTTPAttempts, fetchStats.NetworkBytes, fetchStats.Duration.Milliseconds(), r.nowFunc()) + cancel() + if costErr != nil && ctx.Err() == nil { + log.Printf("source cost observation failed source_id=%s", r.sourceID(spec)) + } + } if truncated && err == nil { r.metrics.AddSourcesTruncated(1) log.Printf("source truncated source=%q limit=%d: ignoring the rest of the feed", sourceLogLabel(spec.URL), r.config.MaxRecordsPerSource) @@ -1086,6 +1164,7 @@ func (r *Runner) workerLoop(ctx context.Context, feed *claimFeed) { r.metrics.RecordClaim() startedProbeAt := time.Now() + probeWorkStarted := time.Now() result := r.runProbe(ctx, claim.URL) r.metrics.AddProbeTraffic(result.UploadBytes, result.DownloadBytes) @@ -1105,6 +1184,7 @@ func (r *Runner) workerLoop(ctx context.Context, feed *claimFeed) { meta := r.probeOutcomeMeta(ctx, claim.URL, result) r.sampleHTTPS(ctx, claim.URL, result, &meta) r.sampleTargetProfileWithinLease(ctx, claim.URL, result, &meta, startedProbeAt) + meta.SourceProbeDurationMS = time.Since(probeWorkStarted).Milliseconds() if !r.commitProbeResult(ctx, claim, result, meta) { return } diff --git a/internal/monitor/source_efficiency.go b/internal/monitor/source_efficiency.go new file mode 100644 index 0000000..3f2571e --- /dev/null +++ b/internal/monitor/source_efficiency.go @@ -0,0 +1,396 @@ +package monitor + +import ( + "context" + "fmt" + "log" + "sort" + "strings" + "sync" + "time" + + "github.com/Crawlora-org/FreeProxyAPI/internal/store" +) + +const ( + sourceEfficiencyCacheTTL = time.Minute + sourceEfficiencyMaxIDs = store.MaxLearningSources + sourceEfficiencyBatch = store.MaxSourceQualityReadBatch + sourceEfficiencyTimeout = 5 * time.Second +) + +type sourceEfficiencyIdentity struct { + ID string + EffectiveScheme string +} + +type sourceEfficiencyQualitySnapshot struct { + identities []sourceEfficiencyIdentity + quality map[string]store.SourceQuality + capturedAt time.Time +} + +// sourceEfficiencyState retains only one bounded current-catalog snapshot. +// The mutex also coalesces concurrent report and stats-publish reads. +type sourceEfficiencyState struct { + mu sync.Mutex + snapshot sourceEfficiencyQualitySnapshot +} + +// sourceEfficiencyMetricHelp is consumed by the Prometheus exposition writer. +// Every source label is a fixed-width hash; source URLs and endpoint data are +// never used as labels. +var sourceEfficiencyMetricHelp = map[string]string{ + "freeproxyapi_source_unattributed_validated": "Validated records with missing or retired source provenance.", + "freeproxyapi_source_unattributed_routing_eligible": "Routing-filter matches with missing or retired source provenance.", + "freeproxyapi_source_admitted_total": "Candidate IDs newly admitted to the tracked set by source.", + "freeproxyapi_source_admission_refreshes_total": "Completed whole-refresh admission observations for source.", + "freeproxyapi_source_http_attempts_total": "HTTP attempts attributed to source fetches.", + "freeproxyapi_source_response_bytes_total": "Response bytes attributed to source fetches.", + "freeproxyapi_source_fetch_work_milliseconds_total": "Fetch duration milliseconds accumulated for source.", + "freeproxyapi_source_probe_completions_total": "Committed candidate probe completions attributed to source.", + "freeproxyapi_source_probe_successful_completions_total": "Committed successful candidate probe completions attributed to source.", + "freeproxyapi_source_probe_work_milliseconds_total": "Committed candidate probe duration milliseconds accumulated for source.", + "freeproxyapi_source_validated": "Currently validated candidates attributed to source.", + "freeproxyapi_source_routing_eligible": "Currently validated candidates meeting live strict router evidence attributed to source.", + "freeproxyapi_source_zero_admission_streak": "Completed consecutive source refresh observations with zero new admissions.", + "freeproxyapi_source_efficiency_quality_snapshot_age_seconds": "Age of the source quality snapshot used for efficiency metrics.", + "freeproxyapi_source_efficiency_counts_snapshot_age_seconds": "Age of the validated-source count snapshot used for efficiency metrics.", + "freeproxyapi_source_efficiency_counts_observed": "Whether source-attributed validated counts were observed in the current shared snapshot.", +} + +var sourceEfficiencyMetricType = map[string]string{ + "freeproxyapi_source_unattributed_validated": "gauge", + "freeproxyapi_source_unattributed_routing_eligible": "gauge", + "freeproxyapi_source_admitted_total": "counter", + "freeproxyapi_source_admission_refreshes_total": "counter", + "freeproxyapi_source_http_attempts_total": "counter", + "freeproxyapi_source_response_bytes_total": "counter", + "freeproxyapi_source_fetch_work_milliseconds_total": "counter", + "freeproxyapi_source_probe_completions_total": "counter", + "freeproxyapi_source_probe_successful_completions_total": "counter", + "freeproxyapi_source_probe_work_milliseconds_total": "counter", + "freeproxyapi_source_validated": "gauge", + "freeproxyapi_source_routing_eligible": "gauge", + "freeproxyapi_source_zero_admission_streak": "gauge", + "freeproxyapi_source_efficiency_quality_snapshot_age_seconds": "gauge", + "freeproxyapi_source_efficiency_counts_snapshot_age_seconds": "gauge", + "freeproxyapi_source_efficiency_counts_observed": "gauge", +} + +type sourceEfficiencyReport struct { + UnattributedValidated *int64 `json:"unattributed_validated,omitempty"` + UnattributedRouting *int64 `json:"unattributed_routing_eligible,omitempty"` + Status string `json:"status"` + CheckedAt string `json:"checked_at"` + QualityCapturedAt string `json:"quality_captured_at,omitempty"` + QualityAgeSeconds *int64 `json:"quality_age_seconds,omitempty"` + QualityStale bool `json:"quality_stale"` + QualityError string `json:"quality_error,omitempty"` + SourceCountsObserved bool `json:"source_counts_observed"` + SourceCountsAt string `json:"source_counts_at,omitempty"` + SourceCountsAgeSeconds *int64 `json:"source_counts_age_seconds,omitempty"` + Sources []sourceEfficiencySourceReport `json:"sources"` +} + +type sourceEfficiencySourceReport struct { + QualityCorrupt bool `json:"quality_corrupt"` + SourceID string `json:"source_id"` + EffectiveScheme string `json:"effective_scheme,omitempty"` + AdmissionMeasured bool `json:"admission_measured"` + Admitted *int64 `json:"admitted,omitempty"` + AdmissionRefreshes *int64 `json:"admission_refreshes,omitempty"` + LastAdmitted *int64 `json:"last_admitted,omitempty"` + ZeroAdmissionStreak *int64 `json:"zero_admission_streak,omitempty"` + LastAdmissionAt string `json:"last_admission_at,omitempty"` + FetchCostMeasured bool `json:"fetch_cost_measured"` + HTTPAttempts *int64 `json:"http_attempts,omitempty"` + NetworkBytes *int64 `json:"network_bytes,omitempty"` + FetchDurationMS *int64 `json:"fetch_duration_ms,omitempty"` + FetchCostObservations *int64 `json:"fetch_cost_observations,omitempty"` + ProbeCostMeasured bool `json:"probe_cost_measured"` + ProbeCompletions *int64 `json:"probe_completions,omitempty"` + ProbeSuccessful *int64 `json:"probe_successful_completions,omitempty"` + ProbeDurationMS *int64 `json:"probe_duration_ms,omitempty"` + Validated *int64 `json:"validated,omitempty"` + RoutingEligible *int64 `json:"routing_eligible,omitempty"` +} + +func (r *Runner) currentSourceEfficiencyIdentities() ([]sourceEfficiencyIdentity, error) { + byID := make(map[string]string, min(len(r.config.SourceSpecs), sourceEfficiencyMaxIDs)) + for _, spec := range r.config.SourceSpecs { + id := r.sourceID(spec) + if _, exists := byID[id]; !exists { + if len(byID) >= sourceEfficiencyMaxIDs { + return nil, fmt.Errorf("source efficiency catalog exceeds bound") + } + byID[id] = strings.ToLower(strings.TrimSpace(spec.defaultScheme(r.config.DefaultProxyScheme))) + } + } + ids := make([]string, 0, len(byID)) + for id := range byID { + ids = append(ids, id) + } + sort.Strings(ids) + identities := make([]sourceEfficiencyIdentity, len(ids)) + for i, id := range ids { + identities[i] = sourceEfficiencyIdentity{ID: id, EffectiveScheme: byID[id]} + } + return identities, nil +} + +func sameSourceEfficiencyIdentities(left, right []sourceEfficiencyIdentity) bool { + if len(left) != len(right) { + return false + } + for i := range left { + if left[i] != right[i] { + return false + } + } + return true +} + +func (r *Runner) loadSourceEfficiencySnapshot(ctx context.Context, now time.Time) (sourceEfficiencyQualitySnapshot, error) { + identities, err := r.currentSourceEfficiencyIdentities() + if err != nil { + return sourceEfficiencyQualitySnapshot{}, err + } + state := &r.sourceEfficiency + if err := lockSourceEfficiency(ctx, &state.mu); err != nil { + return sourceEfficiencyQualitySnapshot{ + identities: append([]sourceEfficiencyIdentity(nil), identities...), + quality: map[string]store.SourceQuality{}, + }, err + } + defer state.mu.Unlock() + if sameSourceEfficiencyIdentities(state.snapshot.identities, identities) && !state.snapshot.capturedAt.IsZero() { + age := now.Sub(state.snapshot.capturedAt) + if age >= 0 && age < sourceEfficiencyCacheTTL { + return state.snapshot, nil + } + } + ids := make([]string, len(identities)) + for i, identity := range identities { + ids[i] = identity.ID + } + redisStore := r.readStore() + if len(identities) == 0 { + snapshot := sourceEfficiencyQualitySnapshot{ + identities: []sourceEfficiencyIdentity{}, + quality: map[string]store.SourceQuality{}, + capturedAt: now, + } + state.snapshot = snapshot + return snapshot, nil + } + if redisStore == nil { + return snapshotOnFailure(state.snapshot, identities), fmt.Errorf("source efficiency store unavailable") + } + quality := make(map[string]store.SourceQuality, len(ids)) + for start := 0; start < len(ids); start += sourceEfficiencyBatch { + end := min(start+sourceEfficiencyBatch, len(ids)) + batch, loadErr := redisStore.LoadSourceQuality(ctx, ids[start:end], now) + if loadErr != nil { + return snapshotOnFailure(state.snapshot, identities), loadErr + } + for id, value := range batch { + quality[id] = value + } + } + snapshot := sourceEfficiencyQualitySnapshot{ + identities: append([]sourceEfficiencyIdentity(nil), identities...), + quality: quality, + capturedAt: now, + } + state.snapshot = snapshot + return snapshot, nil +} + +func lockSourceEfficiency(ctx context.Context, mu *sync.Mutex) error { + ticker := time.NewTicker(5 * time.Millisecond) + defer ticker.Stop() + for !mu.TryLock() { + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + } + } + return nil +} + +func snapshotOnFailure(cached sourceEfficiencyQualitySnapshot, requested []sourceEfficiencyIdentity) sourceEfficiencyQualitySnapshot { + if sameSourceEfficiencyIdentities(cached.identities, requested) { + return cached + } + return sourceEfficiencyQualitySnapshot{ + identities: append([]sourceEfficiencyIdentity(nil), requested...), + quality: map[string]store.SourceQuality{}, + } +} + +func (r *Runner) sourceEfficiencyReport(ctx context.Context) sourceEfficiencyReport { + now := time.Now().UTC() + if r.nowFunc != nil { + now = r.nowFunc() + } + checkedAt := now.UTC().Format(time.RFC3339) + report := sourceEfficiencyReport{Status: "ok", CheckedAt: checkedAt, Sources: []sourceEfficiencySourceReport{}} + if !r.config.SourceLearningEnabled { + report.Status = "disabled" + return report + } + loadCtx, cancel := context.WithTimeout(ctx, sourceEfficiencyTimeout) + defer cancel() + snapshot, err := r.loadSourceEfficiencySnapshot(loadCtx, now) + if err != nil { + report.Status = "unavailable" + report.QualityError = "source quality unavailable" + if len(snapshot.identities) == 0 { + if identities, idErr := r.currentSourceEfficiencyIdentities(); idErr == nil { + snapshot.identities = identities + } + } + } + if !snapshot.capturedAt.IsZero() { + report.QualityCapturedAt = snapshot.capturedAt.UTC().Format(time.RFC3339) + age := now.Sub(snapshot.capturedAt) + if age < 0 { + report.QualityStale = true + age = 0 + } else if age >= sourceEfficiencyCacheTTL { + report.QualityStale = true + } + seconds := int64(age / time.Second) + report.QualityAgeSeconds = &seconds + } else { + report.QualityStale = true + } + slices := r.validatedSlicesSnapshot() + now = time.Now().UTC() + if r.nowFunc != nil { + now = r.nowFunc() + } + report.CheckedAt = now.UTC().Format(time.RFC3339) + countsAt := slices.SourceCountsAtMS + if slices.SourceCountsObserved && countsAt > 0 && countsAt <= now.UnixMilli() { + report.SourceCountsObserved = true + report.UnattributedValidated = int64Pointer(slices.UnattributedValidated) + report.UnattributedRouting = int64Pointer(slices.UnattributedRouting) + report.SourceCountsAt = time.UnixMilli(countsAt).UTC().Format(time.RFC3339) + age := int64((now.UnixMilli() - countsAt) / 1000) + report.SourceCountsAgeSeconds = &age + } + for _, identity := range snapshot.identities { + report.Sources = append(report.Sources, sourceEfficiencyReportEntry(identity, snapshot.quality[identity.ID], r.sourceProbeContract(), slices, report.SourceCountsObserved)) + } + return report +} + +func sourceEfficiencyReportEntry(identity sourceEfficiencyIdentity, quality store.SourceQuality, currentContract string, slices validatedSlices, countsObserved bool) sourceEfficiencySourceReport { + entry := sourceEfficiencySourceReport{QualityCorrupt: quality.Corrupt, SourceID: identity.ID, EffectiveScheme: identity.EffectiveScheme} + if !quality.Corrupt && (quality.AdmissionRefreshes > 0 || quality.Admitted > 0) { + entry.AdmissionMeasured = true + entry.Admitted = int64Pointer(quality.Admitted) + entry.AdmissionRefreshes = int64Pointer(quality.AdmissionRefreshes) + entry.LastAdmitted = int64Pointer(quality.LastAdmitted) + entry.ZeroAdmissionStreak = int64Pointer(quality.ZeroAdmissionStreak) + if !quality.LastAdmissionAt.IsZero() { + entry.LastAdmissionAt = quality.LastAdmissionAt.UTC().Format(time.RFC3339) + } + } + if !quality.Corrupt && quality.FetchCostObservations > 0 { + entry.FetchCostMeasured = true + entry.HTTPAttempts = int64Pointer(quality.HTTPAttempts) + entry.NetworkBytes = int64Pointer(quality.NetworkBytes) + entry.FetchDurationMS = int64Pointer(quality.FetchDurationMS) + entry.FetchCostObservations = int64Pointer(quality.FetchCostObservations) + } + if !quality.Corrupt && quality.ProbeCompletions > 0 && validSourceProbeContract(currentContract) && quality.ProbeContract == currentContract { + entry.ProbeCostMeasured = true + entry.ProbeCompletions = int64Pointer(quality.ProbeCompletions) + entry.ProbeSuccessful = int64Pointer(quality.ProbeSuccessfulCompletions) + entry.ProbeDurationMS = int64Pointer(quality.ProbeDurationMS) + } + if countsObserved { + entry.Validated = int64Pointer(slices.BySource[identity.ID]) + entry.RoutingEligible = int64Pointer(slices.RoutingBySource[identity.ID]) + } + return entry +} + +func validSourceProbeContract(contract string) bool { + if len(contract) != 64 { + return false + } + for _, r := range contract { + if !(r >= '0' && r <= '9') && !(r >= 'a' && r <= 'f') { + return false + } + } + return true +} + +func int64Pointer(value int64) *int64 { return &value } + +// publishSourceEfficiency publishes only the leader's bounded source view. +// Per-source validated counts come from the existing stats-leader scan result; +// this method never scans candidate hashes itself. +func (r *Runner) publishSourceEfficiency(ctx context.Context, slices validatedSlices, now time.Time) { + if r.metrics == nil { + return + } + if !r.config.SourceLearningEnabled || !r.config.StatsLeaderEnabled { + r.metrics.ReplaceSourceGauges(nil) + return + } + if r.config.StatsLeaderEnabled && !r.isStatsLeader() { + r.metrics.ReplaceSourceGauges(nil) + return + } + snapshot, err := r.loadSourceEfficiencySnapshot(ctx, now) + if err != nil && ctx.Err() == nil { + log.Printf("source efficiency snapshot unavailable") + } + gauges := make(map[string]int64) + if !snapshot.capturedAt.IsZero() { + qualityAge := int64(max(0, int64(now.Sub(snapshot.capturedAt)/time.Second))) + gauges["freeproxyapi_source_efficiency_quality_snapshot_age_seconds"] = qualityAge + } + countsObserved := slices.SourceCountsObserved && slices.SourceCountsAtMS > 0 && slices.SourceCountsAtMS <= now.UnixMilli() && now.UnixMilli()-slices.SourceCountsAtMS <= statsAggregateMaxAge.Milliseconds() + if countsObserved { + countsAge := (now.UnixMilli() - slices.SourceCountsAtMS) / 1000 + gauges["freeproxyapi_source_efficiency_counts_snapshot_age_seconds"] = countsAge + gauges["freeproxyapi_source_efficiency_counts_observed"] = 1 + gauges["freeproxyapi_source_unattributed_validated"] = slices.UnattributedValidated + gauges["freeproxyapi_source_unattributed_routing_eligible"] = slices.UnattributedRouting + } else { + gauges["freeproxyapi_source_efficiency_counts_observed"] = 0 + } + contract := r.sourceProbeContract() + for _, identity := range snapshot.identities { + quality := snapshot.quality[identity.ID] + label := fmt.Sprintf("{source_id=%q}", identity.ID) + if !quality.Corrupt && (quality.AdmissionRefreshes > 0 || quality.Admitted > 0) { + gauges["freeproxyapi_source_admitted_total"+label] = quality.Admitted + gauges["freeproxyapi_source_admission_refreshes_total"+label] = quality.AdmissionRefreshes + gauges["freeproxyapi_source_zero_admission_streak"+label] = quality.ZeroAdmissionStreak + } + if !quality.Corrupt && quality.FetchCostObservations > 0 { + gauges["freeproxyapi_source_http_attempts_total"+label] = quality.HTTPAttempts + gauges["freeproxyapi_source_response_bytes_total"+label] = quality.NetworkBytes + gauges["freeproxyapi_source_fetch_work_milliseconds_total"+label] = quality.FetchDurationMS + } + if !quality.Corrupt && quality.ProbeCompletions > 0 && validSourceProbeContract(contract) && quality.ProbeContract == contract { + gauges["freeproxyapi_source_probe_completions_total"+label] = quality.ProbeCompletions + gauges["freeproxyapi_source_probe_successful_completions_total"+label] = quality.ProbeSuccessfulCompletions + gauges["freeproxyapi_source_probe_work_milliseconds_total"+label] = quality.ProbeDurationMS + } + if countsObserved { + gauges["freeproxyapi_source_validated"+label] = slices.BySource[identity.ID] + gauges["freeproxyapi_source_routing_eligible"+label] = slices.RoutingBySource[identity.ID] + } + } + r.metrics.ReplaceSourceGauges(gauges) +} diff --git a/internal/monitor/source_efficiency_integration_test.go b/internal/monitor/source_efficiency_integration_test.go new file mode 100644 index 0000000..2067d13 --- /dev/null +++ b/internal/monitor/source_efficiency_integration_test.go @@ -0,0 +1,128 @@ +package monitor + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/Crawlora-org/FreeProxyAPI/internal/store" +) + +func TestSourceAdmissionCooldownUsesStoredFullRefreshObservations(t *testing.T) { + ctx := context.Background() + now := time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC) + runner, _, clock := newSourceEfficiencyTestRunner(t, []SourceSpec{{URL: "file:///synthetic/feed", Scheme: "http"}}) + *clock = now + id := runner.sourceID(runner.config.SourceSpecs[0]) + candidate := SourceLearningCandidate{ID: id, FallbackPriority: 20} + config := SourceLearningConfig{Enabled: true, MinSamples: 8, ZeroAdmissionCooldown: time.Hour, ZeroAdmissionRefreshes: 3, ExplorationInterval: 10} + for i := 1; i <= 3; i++ { + if err := runner.store.RecordSourceAdmissionRefresh(ctx, map[string]int64{id: 0}, now.Add(time.Duration(i)*time.Minute)); err != nil { + t.Fatal(err) + } + } + quality, err := runner.store.LoadSourceQuality(ctx, []string{id}, now.Add(3*time.Minute)) + if err != nil { + t.Fatal(err) + } + candidate.Quality = quality[id] + plan := BuildSourceLearningPlan([]SourceLearningCandidate{candidate}, 1, now.Add(3*time.Minute), config) + if len(plan) != 0 { + t.Fatalf("three stored zero admissions should cool source: %+v", plan) + } + // The stable exploration slot still admits the cooled feed periodically. + for generation := uint64(1); generation < 20; generation++ { + plan = BuildSourceLearningPlan([]SourceLearningCandidate{candidate}, generation, now.Add(3*time.Minute), config) + if len(plan) == 1 { + if !plan[0].Exploration || !plan[0].AdmissionCooled { + t.Fatalf("exploration plan = %+v", plan[0]) + } + break + } + } + if len(plan) != 1 { + t.Fatal("cooled source was never selected for deterministic exploration") + } + if err := runner.store.RecordSourceAdmissionRefresh(ctx, map[string]int64{id: 1}, now.Add(4*time.Minute)); err != nil { + t.Fatal(err) + } + quality, err = runner.store.LoadSourceQuality(ctx, []string{id}, now.Add(4*time.Minute)) + if err != nil || quality[id].ZeroAdmissionStreak != 0 || quality[id].LastAdmitted != 1 { + t.Fatalf("positive admission did not reset stored streak: %+v err=%v", quality[id], err) + } + candidate.Quality = quality[id] + if plan = BuildSourceLearningPlan([]SourceLearningCandidate{candidate}, 2, now.Add(4*time.Minute), config); len(plan) != 1 || plan[0].AdmissionCooled { + t.Fatalf("positive admission remained cooled: %+v", plan) + } + // Legacy quality with fewer than three measured observations fails open. + if err := runner.store.RecordSourceAdmissionRefresh(ctx, map[string]int64{id: 0}, now.Add(5*time.Minute)); err != nil { + t.Fatal(err) + } + quality, err = runner.store.LoadSourceQuality(ctx, []string{id}, now.Add(5*time.Minute)) + if err != nil { + t.Fatal(err) + } + legacy := SourceLearningCandidate{ID: "legacy-feed", FallbackPriority: 0, Quality: store.SourceQuality{ZeroAdmissionStreak: 2, LastAdmissionAt: now}} + if got := BuildSourceLearningPlan([]SourceLearningCandidate{legacy}, 1, now.Add(5*time.Minute), config); len(got) != 1 || got[0].AdmissionCooled { + t.Fatalf("legacy source with fewer than three observations was cooled: %+v", got) + } +} + +func TestCappedRefreshDoesNotRecordAdmissionObservation(t *testing.T) { + runner, mini, _ := capRunner(t, `, "max_candidates": 1`, feedLines("cap", 12)) + runner.config.MaxCandidates = 0 + if err := runner.refresh(context.Background()); err != nil { + t.Fatalf("initial refresh: %v", err) + } + runner.config.MaxCandidates = 1 + runner.config.SourceLearningEnabled = true + mini.FastForward(time.Hour) + if err := runner.refresh(context.Background()); err != nil { + t.Fatalf("capped refresh: %v", err) + } + id := runner.sourceID(runner.config.SourceSpecs[0]) + quality, err := runner.store.LoadSourceQuality(context.Background(), []string{id}, time.Now()) + if err != nil { + t.Fatal(err) + } + if quality[id].AdmissionRefreshes != 0 { + t.Fatalf("capped source recorded %d admission observations", quality[id].AdmissionRefreshes) + } +} + +func TestSourceReportRouteIsAdminOnly(t *testing.T) { + public, err := startHealthServer("127.0.0.1:0", adminTestRunner(t, Config{AdminListenAddr: "127.0.0.1:0"})) + if err != nil { + t.Fatal(err) + } + defer public.Shutdown() + request := httptest.NewRequest(http.MethodGet, "/report/sources", nil) + response := httptest.NewRecorder() + public.server.Handler.ServeHTTP(response, request) + if response.Code != http.StatusNotFound { + t.Fatalf("public source report status=%d, want 404", response.Code) + } + response = httptest.NewRecorder() + public.admin.Handler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/report/sources", nil)) + if response.Code == http.StatusNotFound { + t.Fatalf("admin source report route missing: %s", response.Body.String()) + } +} + +func TestTruncatedRefreshDoesNotRecordAdmissionObservation(t *testing.T) { + runner, _, _ := capRunner(t, `, "max_records_per_source": 2`, feedLines("truncated", 12)) + runner.config.SourceLearningEnabled = true + if err := runner.refresh(context.Background()); err != nil { + t.Fatalf("refresh: %v", err) + } + id := runner.sourceID(runner.config.SourceSpecs[0]) + quality, err := runner.store.LoadSourceQuality(context.Background(), []string{id}, time.Now()) + if err != nil { + t.Fatal(err) + } + if quality[id].AdmissionRefreshes != 0 { + t.Fatalf("truncated source recorded %d admission observations", quality[id].AdmissionRefreshes) + } +} diff --git a/internal/monitor/source_efficiency_test.go b/internal/monitor/source_efficiency_test.go new file mode 100644 index 0000000..7b4941f --- /dev/null +++ b/internal/monitor/source_efficiency_test.go @@ -0,0 +1,261 @@ +package monitor + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + "time" + + "github.com/Crawlora-org/FreeProxyAPI/internal/store" + "github.com/alicebob/miniredis/v2" +) + +func newSourceEfficiencyTestRunner(t *testing.T, specs []SourceSpec) (*Runner, *miniredis.Miniredis, *time.Time) { + t.Helper() + mini := miniredis.RunT(t) + redisStore, err := store.NewRedis("redis://"+mini.Addr()+"/0", "source-efficiency-test:v1") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = redisStore.Close() }) + now := time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC) + runner := &Runner{ + config: Config{SourceLearningEnabled: true, SourceSpecs: specs, DefaultProxyScheme: "http", StatsLeaderEnabled: true}, + store: redisStore, metrics: newMetrics(), + nowFunc: func() time.Time { return now }, + } + return runner, mini, &now +} + +func TestSourceEfficiencyReportIsBoundedAndDoesNotExposeURLs(t *testing.T) { + ctx := context.Background() + spec := SourceSpec{URL: "https://secret-source.example/feed?token=must-not-escape", Scheme: "http"} + runner, mini, now := newSourceEfficiencyTestRunner(t, []SourceSpec{spec}) + id := runner.sourceID(spec) + if _, err := runner.store.UpsertPrioritizedSourceCounts(ctx, []string{"http://proxy.example:8080"}, *now, id); err != nil { + t.Fatal(err) + } + if err := runner.store.RecordSourceAdmissionRefresh(ctx, map[string]int64{id: 1}, *now); err != nil { + t.Fatal(err) + } + if err := runner.store.RecordSourceFetchCost(ctx, id, 2, 4096, 75, *now); err != nil { + t.Fatal(err) + } + mini.HSet("source-efficiency-test:v1:source-quality:"+id, + "probe_contract", runner.sourceProbeContract(), + "probe_completions", "3", "probe_successful_completions", "2", "probe_duration_ms", "55") + first := runner.sourceEfficiencyReport(ctx) + if first.Status != "ok" || len(first.Sources) != 1 || first.Sources[0].SourceID != id || first.Sources[0].EffectiveScheme != "http" { + t.Fatalf("report identity/status = %+v", first) + } + entry := first.Sources[0] + if !entry.AdmissionMeasured || entry.Admitted == nil || *entry.Admitted != 1 || entry.AdmissionRefreshes == nil || *entry.AdmissionRefreshes != 1 { + t.Fatalf("admission report = %+v", entry) + } + if !entry.FetchCostMeasured || entry.HTTPAttempts == nil || *entry.HTTPAttempts != 2 || entry.NetworkBytes == nil || *entry.NetworkBytes != 4096 { + t.Fatalf("fetch-cost report = %+v", entry) + } + if !entry.ProbeCostMeasured || entry.ProbeCompletions == nil || *entry.ProbeCompletions != 3 || entry.ProbeSuccessful == nil || *entry.ProbeSuccessful != 2 { + t.Fatalf("probe-cost report = %+v", entry) + } + encoded, err := json.Marshal(first) + if err != nil { + t.Fatal(err) + } + for _, forbidden := range []string{"secret-source.example", "must-not-escape", "proxy.example", "http://", "https://"} { + if strings.Contains(string(encoded), forbidden) { + t.Fatalf("report leaked %q: %s", forbidden, encoded) + } + } +} + +func TestSourceEfficiencyLegacyFieldsRemainUnmeasuredAndCountsOmitWhenUnknown(t *testing.T) { + ctx := context.Background() + spec := SourceSpec{URL: "https://source.example/feed", Scheme: "socks5"} + runner, _, now := newSourceEfficiencyTestRunner(t, []SourceSpec{spec}) + id := runner.sourceID(spec) + if err := runner.store.RecordSourceRefreshSuccess(ctx, id, 3, 2, *now); err != nil { + t.Fatal(err) + } + runner.setValidatedSlices(validatedSlices{ + BySource: map[string]int64{id: 4}, RoutingBySource: map[string]int64{id: 3}, + SourceCountsObserved: false, + }) + report := runner.sourceEfficiencyReport(ctx) + if report.Status != "ok" || len(report.Sources) != 1 { + t.Fatalf("report = %+v", report) + } + entry := report.Sources[0] + if entry.AdmissionMeasured || entry.Admitted != nil || entry.FetchCostMeasured || entry.HTTPAttempts != nil || entry.ProbeCostMeasured || entry.ProbeCompletions != nil { + t.Fatalf("legacy counters were presented as measured: %+v", entry) + } + if report.SourceCountsObserved || entry.Validated != nil || entry.RoutingEligible != nil { + t.Fatalf("unobserved source counts were emitted: report=%+v", report) + } +} + +func TestSourceEfficiencyBatchesCatalogReadsAndEnforcesBound(t *testing.T) { + ctx := context.Background() + specs := make([]SourceSpec, store.MaxSourceQualityReadBatch+1) + for i := range specs { + specs[i] = SourceSpec{URL: "https://source-" + strconv.Itoa(i) + ".example/feed", Scheme: "http"} + } + runner, _, _ := newSourceEfficiencyTestRunner(t, specs) + report := runner.sourceEfficiencyReport(ctx) + if report.Status != "ok" || len(report.Sources) != len(specs) { + t.Fatalf("batched source report count=%d status=%q", len(report.Sources), report.Status) + } + encoded, err := json.Marshal(report) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(encoded), "source-0.example") || strings.Contains(string(encoded), "/feed") { + t.Fatal("bounded source report exposed configured feed URL content") + } + tooMany := make([]SourceSpec, sourceEfficiencyMaxIDs+1) + for i := range tooMany { + tooMany[i] = SourceSpec{URL: "https://bounded-" + strconv.Itoa(i) + ".example/feed", Scheme: "http"} + } + runner.config.SourceSpecs = tooMany + bounded := runner.sourceEfficiencyReport(ctx) + if bounded.Status != "unavailable" || bounded.QualityError == "" || len(bounded.Sources) != 0 { + t.Fatalf("oversized source catalog was not rejected: %+v", bounded) + } +} + +func TestSourceEfficiencyReportToleratesUnsetTestClock(t *testing.T) { + runner, _, _ := newSourceEfficiencyTestRunner(t, nil) + runner.nowFunc = nil + report := runner.sourceEfficiencyReport(context.Background()) + if report.Status != "ok" || report.CheckedAt == "" { + t.Fatalf("report with default wall clock = %+v", report) + } +} + +func TestSourceEfficiencyCacheRefreshAndCountFreshness(t *testing.T) { + ctx := context.Background() + spec := SourceSpec{URL: "https://source.example/feed", Scheme: "http"} + runner, _, now := newSourceEfficiencyTestRunner(t, []SourceSpec{spec}) + id := runner.sourceID(spec) + if err := runner.store.RecordSourceFetchCost(ctx, id, 1, 10, 5, *now); err != nil { + t.Fatal(err) + } + first := runner.sourceEfficiencyReport(ctx) + if first.Sources[0].NetworkBytes == nil || *first.Sources[0].NetworkBytes != 10 { + t.Fatalf("initial report = %+v", first.Sources[0]) + } + if err := runner.store.RecordSourceFetchCost(ctx, id, 1, 20, 5, now.Add(time.Second)); err != nil { + t.Fatal(err) + } + *now = now.Add(30 * time.Second) + cached := runner.sourceEfficiencyReport(ctx) + if cached.Sources[0].NetworkBytes == nil || *cached.Sources[0].NetworkBytes != 10 || cached.QualityStale { + t.Fatalf("one-minute cache was not used: %+v", cached) + } + *now = now.Add(31 * time.Second) + refreshed := runner.sourceEfficiencyReport(ctx) + if refreshed.Sources[0].NetworkBytes == nil || *refreshed.Sources[0].NetworkBytes != 30 || refreshed.QualityStale { + t.Fatalf("expired quality cache was not refreshed: %+v", refreshed) + } + runner.setValidatedSlices(validatedSlices{ + BySource: map[string]int64{id: 7}, RoutingBySource: map[string]int64{id: 4}, + SourceCountsObserved: true, SourceCountsAtMS: now.Add(-2 * time.Minute).UnixMilli(), + }) + counts := runner.sourceEfficiencyReport(ctx) + if !counts.SourceCountsObserved || counts.Sources[0].Validated == nil || *counts.Sources[0].Validated != 7 || + counts.Sources[0].RoutingEligible == nil || *counts.Sources[0].RoutingEligible != 4 || counts.SourceCountsAgeSeconds == nil || *counts.SourceCountsAgeSeconds != 120 { + t.Fatalf("observed stale counts lost their value/freshness: %+v", counts) + } +} + +func TestSourceEfficiencyMetricsOnlyPublishOnLeaderAndReplaceStaleSeries(t *testing.T) { + ctx := context.Background() + spec := SourceSpec{URL: "https://source.example/feed", Scheme: "http"} + runner, _, now := newSourceEfficiencyTestRunner(t, []SourceSpec{spec}) + id := runner.sourceID(spec) + if err := runner.store.RecordSourceAdmissionRefresh(ctx, map[string]int64{id: 0}, *now); err != nil { + t.Fatal(err) + } + series := func(metric string) string { return metric + `{source_id="` + id + `"}` } + // Followers must clear old source series instead of exporting stale gauges. + runner.metrics.ReplaceSourceGauges(map[string]int64{series("freeproxyapi_source_admitted_total"): 999}) + runner.publishSourceEfficiency(ctx, validatedSlices{}, *now) + if exposition := metricsExposition(t, runner.metrics.Handler()); strings.Contains(exposition, "freeproxyapi_source_admitted_total") { + t.Fatalf("follower retained source series: %s", exposition) + } + runner.setStatsLeader(true) + runner.setValidatedSlices(validatedSlices{ + BySource: map[string]int64{id: 5}, RoutingBySource: map[string]int64{id: 2}, + SourceCountsObserved: true, SourceCountsAtMS: now.UnixMilli(), + }) + runner.publishSourceEfficiency(ctx, runner.validatedSlicesSnapshot(), *now) + exposition := metricsExposition(t, runner.metrics.Handler()) + for _, want := range []string{ + series("freeproxyapi_source_admitted_total"), + series("freeproxyapi_source_validated"), + series("freeproxyapi_source_routing_eligible"), + `freeproxyapi_source_efficiency_counts_observed 1`, + } { + if !strings.Contains(exposition, want) { + t.Fatalf("source exposition missing %q:\n%s", want, exposition) + } + } + for metric := range sourceEfficiencyMetricHelp { + if sourceEfficiencyMetricType[metric] == "" { + t.Fatalf("metric %q lacks a type", metric) + } + } +} + +func metricsExposition(t *testing.T, handler interface { + ServeHTTP(http.ResponseWriter, *http.Request) +}) string { + t.Helper() + response := httptest.NewRecorder() + handler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/metrics", nil)) + return response.Body.String() +} + +func TestSourceEfficiencyMatchingLegacyContractDoesNotInventProbeCost(t *testing.T) { + r, mini, now := newSourceEfficiencyTestRunner(t, []SourceSpec{{URL: "https://feeds.example/one", Scheme: "http"}}) + id := r.sourceID(r.config.SourceSpecs[0]) + mini.HSet("source-efficiency-test:v1:source-quality:"+id, "probe_contract", r.sourceProbeContract(), "probe_samples", "12", "probe_passes", "1", "probe_updated_ms", strconv.FormatInt(now.UnixMilli(), 10)) + row := r.sourceEfficiencyReport(context.Background()).Sources[0] + if row.ProbeCostMeasured || row.ProbeCompletions != nil { + t.Fatal("matching legacy samples were labeled as measured cost") + } +} +func TestSourceEfficiencyDisabledDoesNotReadOrEmitSources(t *testing.T) { + r, _, _ := newSourceEfficiencyTestRunner(t, []SourceSpec{{URL: "https://feeds.example/one", Scheme: "http"}}) + r.config.SourceLearningEnabled = false + report := r.sourceEfficiencyReport(context.Background()) + if report.Status != "disabled" || len(report.Sources) != 0 { + t.Fatal("disabled instrumentation exposed source metrics") + } +} + +func TestSourceAttributionAggregateIncludesUnattributedAndRetiredRows(t *testing.T) { + spec := SourceSpec{URL: "https://feeds.example/one", Scheme: "http"} + r, mini, now := newSourceEfficiencyTestRunner(t, []SourceSpec{spec}) + sid := r.sourceID(spec) + for _, id := range []string{"known", "retired", "missing"} { + mini.SAdd("source-efficiency-test:v1:validated", id) + mini.HSet("source-efficiency-test:v1:proxy:"+id, "country", "US", "ok_ratio_pct", "100", "success_streak", "3", "https_ok", "1", "https_checked_at_ms", strconv.FormatInt(now.UnixMilli(), 10), "tampered", "0", "tamper_checked_at_ms", strconv.FormatInt(now.UnixMilli(), 10)) + } + mini.HSet("source-efficiency-test:v1:proxy:known", "source_id", sid) + mini.HSet("source-efficiency-test:v1:proxy:retired", "source_id", strings.Repeat("a", 32)) + mini.HSet("source-efficiency-test:v1:proxy:missing", "https_ok", "0") + slices := r.aggregateValidatedSlices(context.Background()) + if slices.BySource[sid] != 1 || slices.UnattributedValidated != 2 || slices.RoutingBySource[sid] != 1 || slices.UnattributedRouting != 1 { + t.Fatalf("attribution=%+v", slices) + } + r.setValidatedSlices(slices) + report := r.sourceEfficiencyReport(context.Background()) + if report.UnattributedValidated == nil || *report.UnattributedValidated != 2 || report.UnattributedRouting == nil || *report.UnattributedRouting != 1 { + t.Fatalf("report=%+v", report) + } +} diff --git a/internal/monitor/source_learning.go b/internal/monitor/source_learning.go index f157361..624f651 100644 --- a/internal/monitor/source_learning.go +++ b/internal/monitor/source_learning.go @@ -18,6 +18,13 @@ type SourceLearningConfig struct { LowYieldThreshold float64 LowYieldCooldown time.Duration ExplorationInterval uint64 + // ZeroAdmissionCooldown suppresses stale, low-confidence feeds that have + // repeatedly completed a refresh without admitting any new candidates. + // Zero disables this policy. + ZeroAdmissionCooldown time.Duration + // ZeroAdmissionRefreshes is the minimum consecutive full refresh outcomes + // required before ZeroAdmissionCooldown can apply. + ZeroAdmissionRefreshes int64 } type SourceLearningCandidate struct { @@ -31,18 +38,20 @@ type SourceLearningCandidate struct { // store.EncodeSourceStageScore. Cold candidates retain the ordering established // by sourcePriority; learned samples affect ordering only after MinSamples. type RankedSource struct { - ID string - Priority int64 - Score float64 - Learned bool - Exploration bool + ID string + Priority int64 + Score float64 + Learned bool + Exploration bool + AdmissionCooled bool } type scoredSource struct { - candidate SourceLearningCandidate - score float64 - learned bool - cooled bool + candidate SourceLearningCandidate + score float64 + learned bool + cooled bool + admissionCooled bool } // StableSourceID shares store's URL-plus-scheme identity so the refresh planner @@ -55,6 +64,9 @@ func StableSourceID(rawURL, defaultScheme string) string { // its effective sample count. A source remains cold until MinSamples, allowing // the configured sourcePriority ordering to remain the fallback. func SourceYield(q store.SourceQuality, now time.Time, config SourceLearningConfig) (score, effectiveSamples float64, learned bool) { + if q.Corrupt { + return 0.5, 0, false + } minimum, halfLife := config.normalized() score, effectiveSamples = store.DecayedSourceYield(q, now, halfLife) return score, effectiveSamples, effectiveSamples >= minimum @@ -80,16 +92,25 @@ func BuildSourceLearningPlan(candidates []SourceLearningCandidate, refreshNumber } seen[candidate.ID] = struct{}{} score, samples := store.DecayedSourceYield(candidate.Quality, now, halfLife) - eligible := config.Enabled && samples >= minimum + if candidate.Quality.Corrupt { + score, samples = 0.5, 0 + } + eligible := config.Enabled && !candidate.Quality.Corrupt && samples >= minimum cooled := false - if config.Enabled { + admissionCooled := false + if config.Enabled && !candidate.Quality.Corrupt { cooled = candidate.Quality.CooldownUntil.After(now) if eligible && config.LowYieldThreshold > 0 && score < config.LowYieldThreshold && config.LowYieldCooldown > 0 { lowYieldUntil := candidate.Quality.ProbeUpdatedAt.Add(config.LowYieldCooldown) cooled = cooled || lowYieldUntil.After(now) } + admissionCooled = shouldCoolZeroAdmission(candidate, now, config) + cooled = cooled || admissionCooled } - scored = append(scored, scoredSource{candidate: candidate, score: score, learned: eligible, cooled: cooled}) + scored = append(scored, scoredSource{ + candidate: candidate, score: score, learned: eligible, cooled: cooled, + admissionCooled: admissionCooled, + }) } if !config.Enabled { sort.SliceStable(scored, func(i, j int) bool { @@ -156,7 +177,7 @@ func BuildSourceLearningPlan(candidates []SourceLearningCandidate, refreshNumber appendSource := func(item scoredSource, isExploration bool) { plan = append(plan, RankedSource{ ID: item.candidate.ID, Priority: int64(len(plan)), Score: item.score, - Learned: item.learned, Exploration: isExploration, + Learned: item.learned, Exploration: isExploration, AdmissionCooled: item.admissionCooled, }) } exploreIndex := 0 @@ -173,6 +194,39 @@ func BuildSourceLearningPlan(candidates []SourceLearningCandidate, refreshNumber return plan } +func shouldCoolZeroAdmission(candidate SourceLearningCandidate, now time.Time, config SourceLearningConfig) bool { + if !config.Enabled || config.ZeroAdmissionCooldown <= 0 || candidate.FallbackPriority <= 10 { + return false + } + refreshes := config.ZeroAdmissionRefreshes + if refreshes < 3 { + refreshes = 3 + } + quality := candidate.Quality + if quality.Corrupt { + return false + } + // A positive observed pass protects a cold but useful feed from the additional novelty cooldown. + if quality.ProbePasses > 0 || quality.ProbeSuccessfulCompletions > 0 { + return false + } + if quality.AdmissionRefreshes < refreshes || quality.ZeroAdmissionStreak < refreshes || quality.LastAdmitted != 0 { + return false + } + if quality.LastAdmissionAt.IsZero() || quality.LastAdmissionAt.After(now) { + return false + } + if !quality.LastAdmissionAt.Add(config.ZeroAdmissionCooldown).After(now) { + return false + } + minimum, halfLife := config.normalized() + score, samples := store.DecayedSourceYield(quality, now, halfLife) + if samples >= minimum && config.LowYieldThreshold >= 0 && score >= config.LowYieldThreshold { + return false + } + return true +} + func (c SourceLearningConfig) normalized() (float64, time.Duration) { minimum := c.MinSamples if minimum <= 0 { diff --git a/internal/monitor/source_learning_runner.go b/internal/monitor/source_learning_runner.go index 100080d..df0b672 100644 --- a/internal/monitor/source_learning_runner.go +++ b/internal/monitor/source_learning_runner.go @@ -28,14 +28,15 @@ type plannedSource struct { priority, score int64 } type sourceLearningReport struct { - Prepared bool `json:"prepared"` - Generation uint64 `json:"generation,omitempty"` - Enabled bool `json:"enabled"` - Configured int `json:"configured"` - Selected int `json:"selected"` - Skipped int `json:"skipped"` - Exploring int `json:"exploring"` - Measured int `json:"measured"` + Prepared bool `json:"prepared"` + Generation uint64 `json:"generation,omitempty"` + Enabled bool `json:"enabled"` + Configured int `json:"configured"` + Selected int `json:"selected"` + Skipped int `json:"skipped"` + Exploring int `json:"exploring"` + Measured int `json:"measured"` + AdmissionCooled int `json:"admission_cooled"` } type sourcePlanState struct { mu sync.RWMutex @@ -89,9 +90,20 @@ func (r *Runner) prepareSourceLearning(ctx context.Context) (int, error) { for _, id := range ids { candidates = append(candidates, SourceLearningCandidate{ID: id, FallbackPriority: sourcePriority(r.config.SourceSpecs[indexes[id]]), Quality: quality[id]}) } - plan := BuildSourceLearningPlan(candidates, generation, now, SourceLearningConfig{Enabled: true, MinSamples: 8, HalfLife: 7 * 24 * time.Hour, LowYieldThreshold: 0.02, LowYieldCooldown: 24 * time.Hour, ExplorationInterval: 10}) + learningConfig := SourceLearningConfig{ + Enabled: true, MinSamples: 8, HalfLife: 7 * 24 * time.Hour, + LowYieldThreshold: 0.02, LowYieldCooldown: 24 * time.Hour, ExplorationInterval: 10, + ZeroAdmissionCooldown: r.config.SourceLearningZeroAdmissionCooldown, + ZeroAdmissionRefreshes: int64(r.config.SourceLearningZeroAdmissionRefreshes), + } + plan := BuildSourceLearningPlan(candidates, generation, now, learningConfig) feeds := make(map[string]plannedSource, len(plan)) report := sourceLearningReport{Enabled: true, Prepared: true, Generation: generation, Configured: len(ids), Selected: len(plan), Skipped: len(ids) - len(plan)} + for _, candidate := range candidates { + if shouldCoolZeroAdmission(candidate, now, learningConfig) { + report.AdmissionCooled++ + } + } for _, item := range plan { index := indexes[item.ID] score, err := store.EncodeSourceStageScore(item.Priority, index) diff --git a/internal/monitor/source_learning_test.go b/internal/monitor/source_learning_test.go index 7044f62..6cf73d3 100644 --- a/internal/monitor/source_learning_test.go +++ b/internal/monitor/source_learning_test.go @@ -173,3 +173,166 @@ func TestSourcePlanDeduplicatesIDs(t *testing.T) { t.Fatalf("deduplicated plan = %+v", plan) } } + +func TestZeroAdmissionCooldownRequiresFreshFullObservations(t *testing.T) { + now := time.Unix(1_800_000_000, 0) + config := SourceLearningConfig{ + Enabled: true, MinSamples: 8, LowYieldThreshold: 0.02, + ZeroAdmissionCooldown: time.Hour, ZeroAdmissionRefreshes: 3, + } + base := SourceLearningCandidate{ID: "duplicate-feed", FallbackPriority: 20} + base.Quality = store.SourceQuality{ + AdmissionRefreshes: 3, + LastAdmitted: 0, + ZeroAdmissionStreak: 3, + LastAdmissionAt: now, + UniqueIncremental: 0, + } + if !shouldCoolZeroAdmission(base, now, config) { + t.Fatal("three completed zero-admission observations should start cooldown for a cold duplicate") + } + + for name, mutate := range map[string]func(*store.SourceQuality){ + "legacy missing observation count": func(q *store.SourceQuality) { q.AdmissionRefreshes = 0 }, + "below consecutive threshold": func(q *store.SourceQuality) { q.AdmissionRefreshes = 2; q.ZeroAdmissionStreak = 2 }, + "corrupt quality fields": func(q *store.SourceQuality) { q.Corrupt = true }, + "latest refresh admitted records": func(q *store.SourceQuality) { + q.LastAdmitted = 1 + q.ZeroAdmissionStreak = 0 + }, + "missing timestamp": func(q *store.SourceQuality) { q.LastAdmissionAt = time.Time{} }, + "future timestamp": func(q *store.SourceQuality) { q.LastAdmissionAt = now.Add(time.Second) }, + "expired cooldown": func(q *store.SourceQuality) { q.LastAdmissionAt = now.Add(-2 * time.Hour) }, + } { + t.Run(name, func(t *testing.T) { + candidate := base + mutate(&candidate.Quality) + if shouldCoolZeroAdmission(candidate, now, config) { + t.Fatal("candidate with untrusted or insufficient observation should fail open") + } + }) + } + if shouldCoolZeroAdmission(base, now, SourceLearningConfig{Enabled: true, ZeroAdmissionRefreshes: 3}) { + t.Fatal("zero cooldown must disable admission cooling") + } + stricter := config + stricter.ZeroAdmissionRefreshes = 5 + if shouldCoolZeroAdmission(base, now, stricter) { + t.Fatal("configured five-refresh threshold must not cool after only three observations") + } +} + +func TestZeroAdmissionCooldownProtectsTrustedAndHighQualityFeeds(t *testing.T) { + now := time.Unix(1_800_000_000, 0) + config := SourceLearningConfig{ + Enabled: true, MinSamples: 8, LowYieldThreshold: 0.02, + ZeroAdmissionCooldown: 24 * time.Hour, ZeroAdmissionRefreshes: 3, + } + quality := store.SourceQuality{ + AdmissionRefreshes: 6, ZeroAdmissionStreak: 6, + LastAdmissionAt: now, LastAdmitted: 0, + } + if shouldCoolZeroAdmission(SourceLearningCandidate{ID: "trusted", FallbackPriority: 10, Quality: quality}, now, config) { + t.Fatal("trusted fallback priority must remain eligible despite duplicate admissions") + } + quality.ProbeSamples, quality.ProbePasses, quality.ProbeUpdatedAt = 16, 16, now + if shouldCoolZeroAdmission(SourceLearningCandidate{ID: "high-quality", FallbackPriority: 20, Quality: quality}, now, config) { + t.Fatal("measured source above the existing low-yield threshold must remain eligible") + } + quality.ProbeSamples, quality.ProbePasses = 128, 0 + if !shouldCoolZeroAdmission(SourceLearningCandidate{ID: "low-quality", FallbackPriority: 20, Quality: quality}, now, config) { + t.Fatal("measured low-yield duplicate should remain eligible for admission cooldown") + } +} + +func TestAdmissionCooledSourcesExploreAndRotate(t *testing.T) { + now := time.Unix(1_800_000_000, 0) + config := SourceLearningConfig{ + Enabled: true, ExplorationInterval: 10, ZeroAdmissionCooldown: time.Hour, + ZeroAdmissionRefreshes: 3, + } + inputs := make([]SourceLearningCandidate, 0, 5) + for i := 0; len(inputs) < 5; i++ { + id := fmt.Sprintf("admission-cooled-%03d", i) + if exploreThisRefresh(id, 0, 10) { + inputs = append(inputs, SourceLearningCandidate{ + ID: id, FallbackPriority: 20, + Quality: store.SourceQuality{ + AdmissionRefreshes: 3, ZeroAdmissionStreak: 3, + LastAdmissionAt: now, + }, + }) + } + } + selected := make(map[string]struct{}, len(inputs)) + for epoch := uint64(0); epoch < uint64(len(inputs)); epoch++ { + plan := BuildSourceLearningPlan(inputs, epoch*10, now, config) + if len(plan) != 1 || !plan[0].Exploration || !plan[0].AdmissionCooled { + t.Fatalf("admission exploration epoch %d plan = %+v", epoch, plan) + } + selected[plan[0].ID] = struct{}{} + } + if len(selected) != len(inputs) { + t.Fatalf("selected %d of %d admission-cooled feeds", len(selected), len(inputs)) + } +} + +func TestAdmissionExplorationDoesNotStarveActivePlan(t *testing.T) { + now := time.Unix(1_800_000_000, 0) + inputs := make([]SourceLearningCandidate, 80) + for i := range inputs { + inputs[i] = SourceLearningCandidate{ID: fmt.Sprintf("active-admission-%03d", i), FallbackPriority: 20} + } + cooled := SourceLearningCandidate{ + ID: "admission-duplicate", FallbackPriority: 20, + Quality: store.SourceQuality{ + AdmissionRefreshes: 3, ZeroAdmissionStreak: 3, LastAdmissionAt: now, + }, + } + inputs = append(inputs, cooled) + cfg := SourceLearningConfig{Enabled: true, ExplorationInterval: 10, ZeroAdmissionCooldown: time.Hour, ZeroAdmissionRefreshes: 3} + var plan []RankedSource + for refresh := uint64(0); refresh < 10; refresh++ { + plan = BuildSourceLearningPlan(inputs, refresh, now, cfg) + if len(plan) > len(inputs)-1 { + break + } + } + if len(plan) != len(inputs) || plan[0].ID != cooled.ID || !plan[0].Exploration || !plan[0].AdmissionCooled { + t.Fatalf("admission exploration was starved or displaced the active plan: %+v", plan[:min(len(plan), 2)]) + } +} + +func TestZeroAdmissionCooldownProtectsColdPositiveProbeEvidence(t *testing.T) { + now := time.Unix(1800000000, 0) + q := store.SourceQuality{AdmissionRefreshes: 3, ZeroAdmissionStreak: 3, LastAdmissionAt: now, ProbeSamples: 1, ProbePasses: 1, ProbeUpdatedAt: now} + cfg := SourceLearningConfig{Enabled: true, ZeroAdmissionCooldown: time.Hour, ZeroAdmissionRefreshes: 3, LowYieldThreshold: 0.02} + if shouldCoolZeroAdmission(SourceLearningCandidate{ID: "positive-cold", FallbackPriority: 100, Quality: q}, now, cfg) { + t.Fatal("cooled useful cold feed") + } +} + +func TestCorruptQualityFailsOpenForEveryLearningCooldown(t *testing.T) { + now := time.Unix(1_800_000_000, 0) + corrupt := store.SourceQuality{ + Corrupt: true, + ProbeSamples: 128, ProbePasses: 0, ProbeUpdatedAt: now, + CooldownUntil: now.Add(time.Hour), + AdmissionRefreshes: 5, ZeroAdmissionStreak: 5, LastAdmissionAt: now, + } + config := SourceLearningConfig{ + Enabled: true, MinSamples: 8, LowYieldThreshold: 0.02, LowYieldCooldown: 24 * time.Hour, + ZeroAdmissionCooldown: time.Hour, ZeroAdmissionRefreshes: 3, + } + if score, samples, learned := SourceYield(corrupt, now, config); learned || score != 0.5 || samples != 0 { + t.Fatalf("corrupt SourceYield = (%0.3f,%0.1f,%t), want unmeasured prior", score, samples, learned) + } + inputs := []SourceLearningCandidate{ + {ID: "checked-fallback", FallbackPriority: 10}, + {ID: "corrupt", FallbackPriority: 80, Quality: corrupt}, + } + plan := BuildSourceLearningPlan(inputs, 1, now, config) + if len(plan) != 2 || plan[0].ID != "checked-fallback" || plan[1].ID != "corrupt" || plan[1].Learned || plan[1].Exploration || plan[1].AdmissionCooled { + t.Fatalf("corrupt source did not fail open at fallback priority: %+v", plan) + } +} diff --git a/internal/monitor/stats_leader.go b/internal/monitor/stats_leader.go index 59d604f..cbc0637 100644 --- a/internal/monitor/stats_leader.go +++ b/internal/monitor/stats_leader.go @@ -34,28 +34,38 @@ type statsLeaderState struct { // sharedValidatedAggregate is the JSON document the stats leader stores in // Redis. It carries only counts, never endpoint identities. type sharedValidatedAggregate struct { - ComputedAtMs int64 `json:"computed_at_ms"` - ByCountry map[string]int64 `json:"by_country"` - StableByCountry map[string]int64 `json:"stable_by_country"` - ByASN map[string]int64 `json:"by_asn"` - ByAnonymity map[string]int64 `json:"by_anonymity"` - ByLatencyBand map[string]int64 `json:"by_latency_band"` - Stable int64 `json:"stable"` - BandKnown int64 `json:"band_known"` - GeoMismatch int64 `json:"geo_mismatch"` + UnattributedValidated int64 `json:"unattributed_validated,omitempty"` + UnattributedRouting int64 `json:"unattributed_routing,omitempty"` + BySource map[string]int64 `json:"by_source,omitempty"` + RoutingBySource map[string]int64 `json:"routing_by_source,omitempty"` + SourceCountsObserved bool `json:"source_counts_observed,omitempty"` + ComputedAtMs int64 `json:"computed_at_ms"` + ByCountry map[string]int64 `json:"by_country"` + StableByCountry map[string]int64 `json:"stable_by_country"` + ByASN map[string]int64 `json:"by_asn"` + ByAnonymity map[string]int64 `json:"by_anonymity"` + ByLatencyBand map[string]int64 `json:"by_latency_band"` + Stable int64 `json:"stable"` + BandKnown int64 `json:"band_known"` + GeoMismatch int64 `json:"geo_mismatch"` } func newSharedValidatedAggregate(v validatedSlices, computedAt time.Time) sharedValidatedAggregate { return sharedValidatedAggregate{ - ComputedAtMs: computedAt.UnixMilli(), - ByCountry: v.ByCountry, - StableByCountry: v.StableByCountry, - ByASN: v.ByASN, - ByAnonymity: v.ByAnonymity, - ByLatencyBand: v.ByLatencyBand, - Stable: v.Stable, - BandKnown: v.BandKnown, - GeoMismatch: v.GeoMismatch, + ComputedAtMs: computedAt.UnixMilli(), + UnattributedValidated: v.UnattributedValidated, + UnattributedRouting: v.UnattributedRouting, + BySource: v.BySource, + RoutingBySource: v.RoutingBySource, + SourceCountsObserved: v.SourceCountsObserved, + ByCountry: v.ByCountry, + StableByCountry: v.StableByCountry, + ByASN: v.ByASN, + ByAnonymity: v.ByAnonymity, + ByLatencyBand: v.ByLatencyBand, + Stable: v.Stable, + BandKnown: v.BandKnown, + GeoMismatch: v.GeoMismatch, } } @@ -67,6 +77,17 @@ func (a sharedValidatedAggregate) slices() validatedSlices { return m } return validatedSlices{ + UnattributedValidated: a.UnattributedValidated, + UnattributedRouting: a.UnattributedRouting, + BySource: orEmpty(a.BySource), + RoutingBySource: orEmpty(a.RoutingBySource), + SourceCountsObserved: a.SourceCountsObserved, + SourceCountsAtMS: func() int64 { + if a.SourceCountsObserved { + return a.ComputedAtMs + } + return 0 + }(), ByCountry: orEmpty(a.ByCountry), StableByCountry: orEmpty(a.StableByCountry), ByASN: orEmpty(a.ByASN), diff --git a/internal/source/fetcher.go b/internal/source/fetcher.go index 4ceec5c..d83614d 100644 --- a/internal/source/fetcher.go +++ b/internal/source/fetcher.go @@ -29,6 +29,18 @@ type Fetcher struct { cacheLen int64 } +// FetchStats describes the work performed by one source stream. NetworkBytes +// counts response-body bytes returned by the HTTP body reader; it excludes +// headers, TLS framing, and cached replay bytes. Duration is wall-clock time +// for the whole StreamMeasured call, including visitor parsing/staging time, +// rather than HTTP latency alone. +type FetchStats struct { + HTTPAttempts int64 + NetworkBytes int64 + Duration time.Duration + CacheHit bool +} + const ( feedCacheMaxBytes int64 = 16 << 20 feedCacheMaxEntries = 128 @@ -108,27 +120,38 @@ func (f *Fetcher) Fetch(ctx context.Context, rawURL string) ([]byte, error) { // larger feeds stream without caching. The callback must consume the reader // before returning. func (f *Fetcher) Stream(ctx context.Context, rawURL string, visit func(io.Reader) error) error { + _, err := f.StreamMeasured(ctx, rawURL, visit) + return err +} + +// StreamMeasured fetches a source and visits its bounded body, returning +// per-call transport and wall-clock measurements. Duration includes the +// visitor's parsing or staging work; NetworkBytes counts only HTTP response +// body bytes actually read, not wire/TLS/header bytes or cached replay bytes. +func (f *Fetcher) StreamMeasured(ctx context.Context, rawURL string, visit func(io.Reader) error) (stats FetchStats, retErr error) { + started := time.Now() + defer func() { stats.Duration = time.Since(started) }() u, err := url.ParseRequestURI(strings.TrimSpace(rawURL)) if err != nil { - return fmt.Errorf("parse source URL: %w", err) + return stats, fmt.Errorf("parse source URL: %w", err) } if visit == nil { - return fmt.Errorf("source visitor is required") + return stats, fmt.Errorf("source visitor is required") } switch u.Scheme { case "file": file, err := os.Open(u.Path) if err != nil { - return fmt.Errorf("open source file: %w", err) + return stats, fmt.Errorf("open source file: %w", err) } defer file.Close() - return f.streamBounded(file, visit) + return stats, f.streamBounded(file, visit) case "http", "https": cacheKey := strings.TrimSpace(rawURL) cached := f.getCached(cacheKey) req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) if err != nil { - return err + return stats, err } // Some public feed providers, including Geonode, reject the default // curl-like agent even though their endpoint is public. @@ -142,26 +165,28 @@ func (f *Fetcher) Stream(ctx context.Context, rawURL string, visit func(io.Reade req.Header.Set("If-Modified-Since", cached.lastModified) } } + stats.HTTPAttempts++ response, err := f.client.Do(req) if err != nil { - return fmt.Errorf("fetch source: %w", err) + return stats, fmt.Errorf("fetch source: %w", err) } defer response.Body.Close() if response.StatusCode == http.StatusNotModified { if cached == nil { - return fmt.Errorf("source returned 304 without a cached body") + return stats, fmt.Errorf("source returned 304 without a cached body") } + stats.CacheHit = true if err := ctx.Err(); err != nil { - return err + return stats, err } err := f.streamBounded(&contextReader{ctx: ctx, reader: bytes.NewReader(cached.body)}, visit) if err != nil { - return err + return stats, err } - return ctx.Err() + return stats, ctx.Err() } if response.StatusCode != http.StatusOK { - return &HTTPStatusError{ + return stats, &HTTPStatusError{ StatusCode: response.StatusCode, Status: response.Status, RetryAfter: parseRetryAfter(response.Header.Get("Retry-After"), time.Now()), @@ -176,39 +201,51 @@ func (f *Fetcher) Stream(ctx context.Context, rawURL string, visit func(io.Reade canCapture := (etag != "" || lastModified != "") && (response.ContentLength < 0 || response.ContentLength <= captureLimit) if !canCapture { - err := f.streamBounded(response.Body, visit) + err := f.streamBounded(&countingReader{reader: response.Body, bytes: &stats.NetworkBytes}, visit) if err != nil { - return err + return stats, err } f.invalidateCached(cacheKey) - return nil + return stats, nil } - captured := &captureReader{reader: &limitReader{reader: response.Body, max: f.maxBytes}, max: captureLimit, cacheable: true} + counted := &countingReader{reader: response.Body, bytes: &stats.NetworkBytes} + captured := &captureReader{reader: &limitReader{reader: counted, max: f.maxBytes}, max: captureLimit, cacheable: true} err = visit(captured) if err != nil { if captured.exceeded { - return captured.limitError(f.maxBytes) + return stats, captured.limitError(f.maxBytes) } - return err + return stats, err } if captured.exceeded { - return captured.limitError(f.maxBytes) + return stats, captured.limitError(f.maxBytes) } if !captured.eof { f.invalidateCached(cacheKey) - return nil + return stats, nil } if captured.cacheable { f.putCached(cacheKey, captured.body, etag, lastModified) } else { f.invalidateCached(cacheKey) } - return nil + return stats, nil default: - return fmt.Errorf("unsupported source URL scheme %q", u.Scheme) + return stats, fmt.Errorf("unsupported source URL scheme %q", u.Scheme) } } +type countingReader struct { + reader io.Reader + bytes *int64 +} + +func (r *countingReader) Read(p []byte) (int, error) { + n, err := r.reader.Read(p) + *r.bytes += int64(n) + return n, err +} + func validCacheValidator(value string) string { value = strings.TrimSpace(value) if value == "" || len(value) > feedCacheValidatorMax { diff --git a/internal/source/fetcher_cost_test.go b/internal/source/fetcher_cost_test.go new file mode 100644 index 0000000..e416d01 --- /dev/null +++ b/internal/source/fetcher_cost_test.go @@ -0,0 +1,151 @@ +package source + +import ( + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestStreamMeasuredHTTPAndConditionalCache(t *testing.T) { + const payload = "203.0.113.1:8080\n" + var requests int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + if r.Header.Get("If-None-Match") == `"v1"` { + w.WriteHeader(http.StatusNotModified) + return + } + w.Header().Set("ETag", `"v1"`) + _, _ = io.WriteString(w, payload) + })) + defer server.Close() + fetcher := newFetcher(time.Second, 1024, plainDialer) + + var first FetchStats + var got string + first, err := fetcher.StreamMeasured(context.Background(), server.URL, func(r io.Reader) error { + b, err := io.ReadAll(r) + got = string(b) + return err + }) + if err != nil || got != payload { + t.Fatalf("first StreamMeasured = (%q, %v)", got, err) + } + if first.HTTPAttempts != 1 || first.NetworkBytes != int64(len(payload)) || first.CacheHit || first.Duration <= 0 { + t.Fatalf("first stats = %+v", first) + } + + second, err := fetcher.StreamMeasured(context.Background(), server.URL, func(r io.Reader) error { + b, err := io.ReadAll(r) + got = string(b) + return err + }) + if err != nil || got != payload { + t.Fatalf("304 StreamMeasured = (%q, %v)", got, err) + } + if second.HTTPAttempts != 1 || second.NetworkBytes != 0 || !second.CacheHit || second.Duration <= 0 { + t.Fatalf("304 stats = %+v", second) + } + if requests != 2 { + t.Fatalf("requests = %d, want 2", requests) + } +} + +func TestStreamMeasuredPartialVisitorFailureCountsConsumedBytes(t *testing.T) { + const payload = "a longer payload that should not be cached after failure" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("ETag", `"v1"`) + _, _ = io.WriteString(w, payload) + })) + defer server.Close() + fetcher := newFetcher(time.Second, 1024, plainDialer) + wantErr := errors.New("parse failed") + stats, err := fetcher.StreamMeasured(context.Background(), server.URL, func(r io.Reader) error { + buf := make([]byte, 3) + _, readErr := io.ReadFull(r, buf) + if readErr != nil { + return readErr + } + return wantErr + }) + if !errors.Is(err, wantErr) { + t.Fatalf("error = %v, want visitor error", err) + } + if stats.HTTPAttempts != 1 || stats.NetworkBytes != 3 || stats.CacheHit || stats.Duration <= 0 { + t.Fatalf("stats = %+v, want one attempt and 3 consumed bytes", stats) + } + if len(fetcher.cache) != 0 { + t.Fatal("failed visitor populated cache") + } +} + +func TestStreamMeasuredOversizeCountsLimitProbe(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("ETag", `"large"`) + _, _ = io.WriteString(w, strings.Repeat("x", 32)) + })) + defer server.Close() + fetcher := newFetcher(time.Second, 8, plainDialer) + stats, err := fetcher.StreamMeasured(context.Background(), server.URL, func(r io.Reader) error { + _, err := io.ReadAll(r) + return err + }) + if !errors.Is(err, ErrSourceTooLarge) { + t.Fatalf("error = %v, want ErrSourceTooLarge", err) + } + if stats.HTTPAttempts != 1 || stats.NetworkBytes != 9 || stats.CacheHit { + t.Fatalf("stats = %+v, want one attempt and limit+1 body bytes", stats) + } +} + +func TestStreamMeasuredRedirectAndUnsupportedScheme(t *testing.T) { + var targetRequests int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/target" { + targetRequests++ + _, _ = io.WriteString(w, "unexpected") + return + } + http.Redirect(w, r, "/target", http.StatusFound) + })) + defer server.Close() + fetcher := newFetcher(time.Second, 1024, plainDialer) + stats, err := fetcher.StreamMeasured(context.Background(), server.URL, func(r io.Reader) error { _, err := io.Copy(io.Discard, r); return err }) + if err == nil || stats.HTTPAttempts != 1 || stats.NetworkBytes != 0 || stats.CacheHit { + t.Fatalf("redirect = stats %+v, error %v", stats, err) + } + if targetRequests != 0 { + t.Fatalf("redirect target requests = %d, want 0", targetRequests) + } + stats, err = fetcher.StreamMeasured(context.Background(), "ftp://inventory.example/list", func(io.Reader) error { return nil }) + if err == nil || stats.HTTPAttempts != 0 || stats.NetworkBytes != 0 || stats.CacheHit { + t.Fatalf("unsupported scheme = stats %+v, error %v", stats, err) + } +} + +func TestStreamMeasuredFileDoesNotCountNetwork(t *testing.T) { + path := filepath.Join(t.TempDir(), "feed.txt") + if err := os.WriteFile(path, []byte("local feed"), 0o600); err != nil { + t.Fatal(err) + } + fetcher := NewFetcher(time.Second, 1024) + var got string + stats, err := fetcher.StreamMeasured(context.Background(), "file://"+path, func(r io.Reader) error { + b, err := io.ReadAll(r) + got = string(b) + return err + }) + if err != nil || got != "local feed" { + t.Fatalf("file stream = (%q, %v)", got, err) + } + if stats.HTTPAttempts != 0 || stats.NetworkBytes != 0 || stats.CacheHit || stats.Duration <= 0 { + t.Fatalf("file stats = %+v", stats) + } +} diff --git a/internal/store/redis.go b/internal/store/redis.go index e1fe103..2e020e4 100644 --- a/internal/store/redis.go +++ b/internal/store/redis.go @@ -400,6 +400,25 @@ local now_ms = tonumber(ARGV[1]) local last_pair = #ARGV local source_id = nil if #ARGV % 2 == 0 then source_id = ARGV[#ARGV]; last_pair = #ARGV - 1 end +local quality_key = nil +local record_admission = false +if source_id then + quality_key = KEYS[7] + -- Efficiency telemetry must never prevent candidate imports. Only enable + -- its increment when the existing hash field is a safe integer with room + -- for every possible insertion in this bounded batch. + local current_raw = redis.pcall('HGET', quality_key, 'admitted') + if not (type(current_raw) == 'table' and current_raw.err) then + if current_raw == false then current_raw = '0' end + local current = tonumber(current_raw) + local max_batch = math.floor((last_pair - 1) / 2) + if current and string.match(current_raw, '^%d+$') and current >= 0 and + current == math.floor(current) and current + max_batch <= 9007199254740991 then + record_admission = true + end + end +end +local source_admitted = 0 for i = 2, last_pair, 2 do local id = ARGV[i] local key = KEYS[4] .. id @@ -412,7 +431,9 @@ for i = 2, last_pair, 2 do if blocked then skipped = skipped + 1 else - added = added + redis.call('SADD', KEYS[1], id) + local inserted = redis.call('SADD', KEYS[1], id) + added = added + inserted + source_admitted = source_admitted + inserted redis.call('HSET', key, 'url', ARGV[i + 1]) if source_id then local previous_source = redis.call('HGET', key, 'source_id') @@ -434,6 +455,10 @@ for i = 2, last_pair, 2 do end end end +if source_id and record_admission and source_admitted > 0 then + redis.call('HINCRBY', quality_key, 'admitted', tostring(source_admitted)) + redis.call('PEXPIRE', quality_key, '2592000000') +end return {added, skipped} `) @@ -473,8 +498,11 @@ func (s *Redis) upsertCounts(ctx context.Context, urls []string, now time.Time, if sourceID != "" { args = append(args, sourceID) } - values, err := upsertScript.Run(ctx, s.client, - []string{s.candidatesKey(), s.pendingKey(), s.leasedKey(), s.prefix + ":proxy:", s.tombstonePrefix(), s.SourceQualityKeyPrefix() + "active"}, args...).Int64Slice() + keys := []string{s.candidatesKey(), s.pendingKey(), s.leasedKey(), s.prefix + ":proxy:", s.tombstonePrefix(), s.SourceQualityKeyPrefix() + "active"} + if sourceID != "" { + keys = append(keys, s.sourceQualityKey(sourceID)) + } + values, err := upsertScript.Run(ctx, s.client, keys, args...).Int64Slice() if err != nil { return UpsertCounts{}, err } @@ -1046,6 +1074,9 @@ type OutcomeMeta struct { // tamper_checked_at_ms); when false the stored tamper fields are unchanged. TamperChecked bool Tampered bool + // SourceProbeDurationMS is the committed probe's elapsed time for source + // efficiency learning. Invalid values suppress only this optional metric. + SourceProbeDurationMS int64 } func (m OutcomeMeta) httpsArgs(now time.Time) (ok, checkedAt string) { @@ -1096,7 +1127,8 @@ func (s *Redis) Complete(ctx context.Context, claim Claim, now time.Time, policy httpsOK, httpsCheckedAt, meta.HTTPSTunnelScheme, tampered, tamperCheckedAt, exitASN, exitOrg, exitHosting, - boolArg(meta.SourceLearning), 16, (7 * 24 * time.Hour).Milliseconds(), encodeTargetProfileUpdate(meta.TargetProfiles), meta.SourceLearningContract).Int() + boolArg(meta.SourceLearning), 16, (7 * 24 * time.Hour).Milliseconds(), encodeTargetProfileUpdate(meta.TargetProfiles), meta.SourceLearningContract, + sourceProbeDurationArg(meta.SourceProbeDurationMS)).Int() if err != nil { return CompleteResult{}, err } @@ -1152,13 +1184,19 @@ func boolArg(value bool) string { // ValidatedInfo is a slice of one validated candidate's routing-relevant // attributes. It carries no endpoint identity. type ValidatedInfo struct { - Country string - ExitCountry string - ASN string - Anonymity string - LatencyMs int64 - JitterMs int64 - OkRatioPct int64 + SourceID string + SuccessStreak *int64 + HTTPSOK *bool + HTTPSCheckedAt int64 + Tampered *bool + TamperCheckedAt int64 + Country string + ExitCountry string + ASN string + Anonymity string + LatencyMs int64 + JitterMs int64 + OkRatioPct int64 } // ValidatedDetails returns country/latency/ratio attributes for every @@ -1172,14 +1210,16 @@ func (s *Redis) ValidatedDetails(ctx context.Context) ([]ValidatedInfo, error) { for _, id := range ids { cmds = append(cmds, pipe.HMGet(ctx, s.proxyKey(id), "country", "exit_country", "asn", "anonymity", - "latency_ewma_ms", "jitter_ewma_ms", "ok_ratio_pct")) + "latency_ewma_ms", "jitter_ewma_ms", "ok_ratio_pct", + "source_id", "success_streak", "https_ok", "https_checked_at_ms", + "tampered", "tamper_checked_at_ms")) } if _, err := pipe.Exec(ctx); err != nil { return err } for _, cmd := range cmds { values, err := cmd.Result() - if err != nil || len(values) != 7 || allNil(values) { + if err != nil || len(values) != 13 || allNil(values) { // All-nil means the hash was deleted (evicted) after the scan; // counting it would report a ghost candidate. continue @@ -1206,6 +1246,28 @@ func (s *Redis) ValidatedDetails(ctx context.Context) ([]ValidatedInfo, error) { if v, _ := values[6].(string); v != "" { info.OkRatioPct, _ = strconv.ParseInt(v, 10, 64) } + if v, _ := values[7].(string); validSourceID(v) { + info.SourceID = v + } + if v, _ := values[8].(string); v != "" { + if streak, parseErr := strconv.ParseInt(v, 10, 64); parseErr == nil && streak >= 0 { + info.SuccessStreak = &streak + } + } + if v, _ := values[9].(string); v == "0" || v == "1" { + passed := v == "1" + info.HTTPSOK = &passed + } + if v, _ := values[10].(string); v != "" { + info.HTTPSCheckedAt, _ = strconv.ParseInt(v, 10, 64) + } + if v, _ := values[11].(string); v == "0" || v == "1" { + tampered := v == "1" + info.Tampered = &tampered + } + if v, _ := values[12].(string); v != "" { + info.TamperCheckedAt, _ = strconv.ParseInt(v, 10, 64) + } out = append(out, info) } return nil diff --git a/internal/store/source_efficiency.go b/internal/store/source_efficiency.go new file mode 100644 index 0000000..071c95a --- /dev/null +++ b/internal/store/source_efficiency.go @@ -0,0 +1,22 @@ +package store + +import "time" + +// RoutingEligible reports whether an already-validated record has the strict +// evidence required by the live GOST router. Callers obtain these records from +// ValidatedDetails, so set membership is implied. A non-positive maxAge removes +// the age bound but never trusts missing or future timestamps. +func (v ValidatedInfo) RoutingEligible(now time.Time, maxAge time.Duration) bool { + nowMS := now.UnixMilli() + if nowMS <= 0 || v.OkRatioPct < 80 || v.SuccessStreak == nil || *v.SuccessStreak < 1 { + return false + } + fresh := func(checkedAt int64) bool { + if checkedAt <= 0 || checkedAt > nowMS { + return false + } + return maxAge <= 0 || nowMS-checkedAt <= maxAge.Milliseconds() + } + return v.HTTPSOK != nil && *v.HTTPSOK && fresh(v.HTTPSCheckedAt) && + v.Tampered != nil && !*v.Tampered && fresh(v.TamperCheckedAt) +} diff --git a/internal/store/source_efficiency_test.go b/internal/store/source_efficiency_test.go new file mode 100644 index 0000000..761cc64 --- /dev/null +++ b/internal/store/source_efficiency_test.go @@ -0,0 +1,460 @@ +package store + +import ( + "context" + "fmt" + "strings" + "testing" + "time" +) + +func TestSourceFetchCostRecordsBoundedAggregates(t *testing.T) { + ctx := context.Background() + _, s := newTestStore(t, "source-fetch-cost:v1") + id := StableSourceID("https://feeds.example/cost", "http") + now := time.Unix(1_800_000_000, 0) + if err := s.RecordSourceFetchCost(ctx, id, 2, 4096, 125, now); err != nil { + t.Fatal(err) + } + if err := s.RecordSourceFetchCost(ctx, id, 1, 1024, 75, now.Add(time.Second)); err != nil { + t.Fatal(err) + } + quality, err := s.LoadSourceQuality(ctx, []string{id}, now.Add(time.Second)) + if err != nil { + t.Fatal(err) + } + got := quality[id] + if got.HTTPAttempts != 3 || got.NetworkBytes != 5120 || got.FetchDurationMS != 200 || got.FetchCostObservations != 2 { + t.Fatalf("fetch cost = %+v", got) + } + if ttl := miniTTL(t, s.client, s.sourceQualityKey(id)); ttl <= 0 || ttl > sourceQualityTTL { + t.Fatalf("source quality TTL = %s", ttl) + } + for _, input := range []struct { + attempts, bytes, duration int64 + }{ + {-1, 0, 0}, {0, -1, 0}, {0, 0, -1}, + {maxSourceHTTPAttempts + 1, 0, 0}, {0, maxSourceNetworkBytes + 1, 0}, + {0, 0, maxSourceCostDurationMS + 1}, + } { + if err := s.RecordSourceFetchCost(ctx, id, input.attempts, input.bytes, input.duration, now); err == nil { + t.Fatalf("accepted invalid fetch cost %+v", input) + } + } + quality, err = s.LoadSourceQuality(ctx, []string{id}, now.Add(time.Second)) + if err != nil || quality[id].HTTPAttempts != 3 || quality[id].FetchCostObservations != 2 { + t.Fatalf("invalid observations changed counters: quality=%+v err=%v", quality[id], err) + } +} + +func TestSourceAdmissionCountsOnlyActualInsertionsAndTracksCompletedRefreshes(t *testing.T) { + ctx := context.Background() + mini, s := newTestStore(t, "source-admission:v1") + now := time.Unix(1_800_000_000, 0) + first := StableSourceID("https://feeds.example/first", "http") + second := StableSourceID("https://feeds.example/second", "http") + if err := s.SetActiveLearningSources(ctx, []string{first, second}); err != nil { + t.Fatal(err) + } + urls := []string{"http://one.example:8080", "http://two.example:8080"} + counts, err := s.UpsertPrioritizedSourceCounts(ctx, urls, now, first) + if err != nil || counts.Added != 2 { + t.Fatalf("initial source upsert=%+v err=%v", counts, err) + } + // Reimporting present candidates updates their schedule/provenance but does + // not count as a new admission. + counts, err = s.UpsertPrioritizedSourceCounts(ctx, urls, now.Add(time.Second), first) + if err != nil || counts.Added != 0 { + t.Fatalf("duplicate source upsert=%+v err=%v", counts, err) + } + claim, ok, err := s.ClaimDue(ctx, "worker", now.Add(time.Second), time.Minute) + if err != nil || !ok { + t.Fatalf("claim before leased duplicate=(%+v,%t,%v)", claim, ok, err) + } + counts, err = s.UpsertPrioritizedSourceCounts(ctx, []string{urls[0]}, now.Add(2*time.Second), first) + if err != nil || counts.Added != 0 { + t.Fatalf("leased duplicate upsert=%+v err=%v", counts, err) + } + leaseToken, err := s.client.HGet(ctx, s.proxyKey(claim.ID), "lease_token").Result() + if err != nil || leaseToken != claim.Token { + t.Fatalf("lease token after duplicate admission=%q err=%v", leaseToken, err) + } + blocked := "http://blocked.example:8080" + if err := mini.Set(s.tombstoneKey(proxyID(blocked)), fmt.Sprintf("1:%d", now.Add(time.Hour).UnixMilli())); err != nil { + t.Fatal(err) + } + counts, err = s.UpsertPrioritizedSourceCounts(ctx, []string{blocked}, now, second) + if err != nil || counts.Added != 0 || counts.Tombstoned != 1 { + t.Fatalf("tombstoned source upsert=%+v err=%v", counts, err) + } + quality, err := s.LoadSourceQuality(ctx, []string{first, second}, now) + if err != nil || quality[first].Admitted != 2 || quality[second].Admitted != 0 { + t.Fatalf("actual admissions first=%+v second=%+v err=%v", quality[first], quality[second], err) + } + + // A completed-refresh report updates the observed latest values and zero + // streaks; it does not double-increment the actual SADD-derived total. + if err := s.RecordSourceAdmissionRefresh(ctx, map[string]int64{first: 2, second: 0}, now.Add(2*time.Second)); err != nil { + t.Fatal(err) + } + if err := s.RecordSourceAdmissionRefresh(ctx, map[string]int64{first: 0, second: 0}, now.Add(3*time.Second)); err != nil { + t.Fatal(err) + } + if err := s.RecordSourceAdmissionRefresh(ctx, map[string]int64{first: 1}, now.Add(4*time.Second)); err != nil { + t.Fatal(err) + } + quality, err = s.LoadSourceQuality(ctx, []string{first, second}, now.Add(4*time.Second)) + if err != nil { + t.Fatal(err) + } + if got := quality[first]; got.Admitted != 2 || got.AdmissionRefreshes != 3 || got.LastAdmitted != 1 || got.ZeroAdmissionStreak != 0 || !got.LastAdmissionAt.Equal(now.Add(4*time.Second)) { + t.Fatalf("first admission state = %+v", got) + } + if got := quality[second]; got.Admitted != 0 || got.AdmissionRefreshes != 2 || got.LastAdmitted != 0 || got.ZeroAdmissionStreak != 2 || !got.LastAdmissionAt.Equal(now.Add(3*time.Second)) { + t.Fatalf("second admission state = %+v", got) + } + if ttl := miniTTL(t, s.client, s.sourceQualityKey(first)); ttl <= 0 || ttl > sourceQualityTTL { + t.Fatalf("source quality TTL = %s", ttl) + } +} + +func TestCorruptAdmissionTelemetryNeverBlocksCandidateImports(t *testing.T) { + ctx := context.Background() + mini, s := newTestStore(t, "source-admission-corrupt:v1") + now := time.Unix(1_800_000_000, 0) + id := StableSourceID("https://feeds.example/corrupt-admission", "http") + if err := s.SetActiveLearningSources(ctx, []string{id}); err != nil { + t.Fatal(err) + } + + tests := []struct { + name string + seed func(key string) + wantStored string + }{ + { + name: "malformed counter", + seed: func(key string) { mini.HSet(key, "admitted", "not-an-integer") }, + wantStored: "not-an-integer", + }, + { + name: "counter at exact cap", + seed: func(key string) { mini.HSet(key, "admitted", fmt.Sprint(int64(1<<53-1))) }, + wantStored: fmt.Sprint(int64(1<<53 - 1)), + }, + { + name: "wrong-type quality key", + seed: func(key string) { + if err := mini.Set(key, "legacy non-hash value"); err != nil { + t.Fatal(err) + } + }, + wantStored: "legacy non-hash value", + }, + } + for index, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + url := fmt.Sprintf("http://admission-%d.example:8080", index) + idKey := s.sourceQualityKey(id) + mini.Del(idKey) + tc.seed(idKey) + counts, err := s.UpsertPrioritizedSourceCounts(ctx, []string{url}, now.Add(time.Duration(index)*time.Second), id) + if err != nil || counts.Added != 1 || counts.Tombstoned != 0 { + t.Fatalf("corrupt telemetry blocked upsert: counts=%+v err=%v", counts, err) + } + candidateID := proxyID(url) + exists, err := s.client.SIsMember(ctx, s.candidatesKey(), candidateID).Result() + if err != nil || !exists { + t.Fatalf("candidate membership=%t err=%v", exists, err) + } + if due, err := s.client.ZScore(ctx, s.pendingKey(), candidateID).Result(); err != nil || due != float64(now.Add(time.Duration(index)*time.Second).UnixMilli()) { + t.Fatalf("pending score=%v err=%v", due, err) + } + if got, err := s.client.HGet(ctx, s.proxyKey(candidateID), "source_id").Result(); err != nil || got != id { + t.Fatalf("candidate provenance=%q err=%v", got, err) + } + var stored string + if tc.name == "wrong-type quality key" { + stored, err = mini.Get(idKey) + } else { + stored = mini.HGet(idKey, "admitted") + } + if err != nil || stored != tc.wantStored { + t.Fatalf("corrupt admission counter changed: got=%q err=%v want=%q", stored, err, tc.wantStored) + } + }) + } +} + +func TestLegacySourceQualityHasUnmeasuredEfficiencyFields(t *testing.T) { + quality := decodeSourceQuality(map[string]string{"refresh_fetches": "1"}, time.Unix(1_800_000_000, 0)) + if quality.Admitted != 0 || quality.AdmissionRefreshes != 0 || quality.LastAdmitted != 0 || quality.ZeroAdmissionStreak != 0 || !quality.LastAdmissionAt.IsZero() || + quality.HTTPAttempts != 0 || quality.NetworkBytes != 0 || quality.FetchDurationMS != 0 || quality.FetchCostObservations != 0 || + quality.ProbeCompletions != 0 || quality.ProbeSuccessfulCompletions != 0 || quality.ProbeDurationMS != 0 || quality.Corrupt { + t.Fatalf("legacy hash should decode efficiency values as unmeasured zeroes: %+v", quality) + } +} + +func TestSourceAdmissionRefreshRejectsInvalidBatchWithoutPartialWrites(t *testing.T) { + ctx := context.Background() + _, s := newTestStore(t, "source-admission-invalid:v1") + now := time.Unix(1_800_000_000, 0) + valid := StableSourceID("https://feeds.example/valid", "http") + if err := s.RecordSourceAdmissionRefresh(ctx, map[string]int64{valid: 1}, now); err != nil { + t.Fatal(err) + } + for _, counts := range []map[string]int64{ + {valid: -1}, {"invalid": 1}, + } { + if err := s.RecordSourceAdmissionRefresh(ctx, counts, now.Add(time.Second)); err == nil { + t.Fatalf("accepted invalid admission report %v", counts) + } + } + tooMany := make(map[string]int64, MaxSourceQualityReadBatch+1) + for i := 0; i <= MaxSourceQualityReadBatch; i++ { + tooMany[StableSourceID(fmt.Sprintf("https://feeds.example/%d", i), "http")] = 0 + } + if err := s.RecordSourceAdmissionRefresh(ctx, tooMany, now.Add(time.Second)); err == nil { + t.Fatal("accepted oversized admission batch") + } + quality, err := s.LoadSourceQuality(ctx, []string{valid}, now) + if err != nil || quality[valid].AdmissionRefreshes != 1 || quality[valid].LastAdmitted != 1 { + t.Fatalf("invalid reports partially changed state: quality=%+v err=%v", quality[valid], err) + } +} + +func TestEfficiencyCounterPreflightRejectsExponentCountersWithoutPartialWrites(t *testing.T) { + ctx := context.Background() + mini, s := newTestStore(t, "source-efficiency-exponent:v1") + now := time.Unix(1_800_000_000, 0) + id := StableSourceID("https://feeds.example/cost-exponent", "http") + key := s.sourceQualityKey(id) + mini.HSet(key, "http_attempts", "11", "network_bytes", "1e3") + if err := s.RecordSourceFetchCost(ctx, id, 1, 10, 1, now); err == nil { + t.Fatal("accepted exponential-notation fetch-cost counter") + } + if got := mini.HGet(key, "http_attempts"); got != "11" { + t.Fatalf("fetch preflight partially incremented earlier field: %q", got) + } + if got := mini.HGet(key, "network_bytes"); got != "1e3" { + t.Fatalf("fetch preflight rewrote malformed field: %q", got) + } + if got := mini.HGet(key, "fetch_cost_observations"); got != "" { + t.Fatalf("fetch preflight created observations field: %q", got) + } + + first := StableSourceID("https://feeds.example/admission-first", "http") + second := StableSourceID("https://feeds.example/admission-second", "http") + goodID, badID := first, second + if goodID > badID { + goodID, badID = badID, goodID + } + mini.HSet(s.sourceQualityKey(badID), "admission_refreshes", "1e3") + if err := s.RecordSourceAdmissionRefresh(ctx, map[string]int64{goodID: 1, badID: 0}, now); err == nil { + t.Fatal("accepted exponential-notation admission counter") + } + if got := mini.HGet(s.sourceQualityKey(goodID), "admission_refreshes"); got != "" { + t.Fatalf("admission preflight partially observed earlier key: %q", got) + } + if got := mini.HGet(s.sourceQualityKey(badID), "admission_refreshes"); got != "1e3" { + t.Fatalf("admission preflight rewrote malformed field: %q", got) + } +} + +func TestCommittedProbeCostIsLeaseGuardedAndContractScoped(t *testing.T) { + ctx := context.Background() + _, s := newTestStore(t, "source-probe-cost:v1") + now := time.Unix(1_800_000_000, 0) + url := "http://probe-cost.example:8080" + id := StableSourceID("https://feeds.example/probe-cost", "http") + contractA := strings.Repeat("a", 64) + contractB := strings.Repeat("b", 64) + if err := s.SetActiveLearningSources(ctx, []string{id}); err != nil { + t.Fatal(err) + } + if _, err := s.UpsertPrioritizedSourceCounts(ctx, []string{url}, now, id); err != nil { + t.Fatal(err) + } + claim, ok, err := s.ClaimDue(ctx, "worker", now, time.Minute) + if err != nil || !ok { + t.Fatalf("claim=(%+v,%t,%v)", claim, ok, err) + } + if _, err := s.Complete(ctx, claim, now, RetestPolicy{ValidatedAfter: time.Hour, FailedAfter: time.Hour}, true, 200, time.Millisecond, "", OutcomeMeta{ + SourceLearning: true, SourceLearningContract: contractA, SourceProbeDurationMS: 17, + }); err != nil { + t.Fatal(err) + } + // Wrong lease tokens cannot update efficiency costs. + if _, err := s.UpsertPrioritizedSourceCounts(ctx, []string{url}, now.Add(time.Hour), id); err != nil { + t.Fatal(err) + } + claim, ok, err = s.ClaimDue(ctx, "worker", now.Add(time.Hour), time.Minute) + if err != nil || !ok { + t.Fatalf("second claim=(%+v,%t,%v)", claim, ok, err) + } + stale := claim + stale.Token = "wrong-token" + result, err := s.Complete(ctx, stale, now.Add(time.Hour), RetestPolicy{ValidatedAfter: time.Hour, FailedAfter: time.Hour}, false, 502, time.Millisecond, "", OutcomeMeta{ + SourceLearning: true, SourceLearningContract: contractA, SourceProbeDurationMS: 50, + }) + if err != nil || result.Committed { + t.Fatalf("stale completion=%+v err=%v", result, err) + } + // A contract change resets both sampled yield and probe-cost counters, then + // records the current committed result under the new contract. + result, err = s.Complete(ctx, claim, now.Add(time.Hour), RetestPolicy{ValidatedAfter: time.Hour, FailedAfter: time.Hour}, false, 502, time.Millisecond, "", OutcomeMeta{ + SourceLearning: true, SourceLearningContract: contractB, SourceProbeDurationMS: 23, + }) + if err != nil || !result.Committed { + t.Fatalf("contract-change completion=%+v err=%v", result, err) + } + quality, err := s.LoadSourceQuality(ctx, []string{id}, now.Add(time.Hour)) + if err != nil { + t.Fatal(err) + } + got := quality[id] + if got.ProbeContract != contractB || got.ProbeCompletions != 1 || got.ProbeSuccessfulCompletions != 0 || got.ProbeDurationMS != 23 || got.ProbeSamples != 1 || got.ProbePasses != 0 { + t.Fatalf("post-contract probe quality=%+v", got) + } + if ttl := miniTTL(t, s.client, s.sourceQualityKey(id)); ttl <= 0 || ttl > sourceQualityTTL { + t.Fatalf("source quality TTL = %s", ttl) + } + // Invalid duration metadata must not block a completed probe or alter cost + // counters. It is omitted rather than leaving the leased candidate stuck. + if _, err := s.UpsertPrioritizedSourceCounts(ctx, []string{url}, now.Add(2*time.Hour), id); err != nil { + t.Fatal(err) + } + claim, ok, err = s.ClaimDue(ctx, "worker", now.Add(2*time.Hour), time.Minute) + if err != nil || !ok { + t.Fatalf("invalid-duration claim=(%+v,%t,%v)", claim, ok, err) + } + result, err = s.Complete(ctx, claim, now.Add(2*time.Hour), RetestPolicy{ValidatedAfter: time.Hour, FailedAfter: time.Hour}, true, 200, time.Millisecond, "", OutcomeMeta{ + SourceLearning: true, SourceLearningContract: contractB, SourceProbeDurationMS: maxSourceCostDurationMS + 1, + }) + if err != nil || !result.Committed { + t.Fatalf("invalid-duration completion=%+v err=%v", result, err) + } + quality, err = s.LoadSourceQuality(ctx, []string{id}, now.Add(2*time.Hour)) + if err != nil || quality[id].ProbeCompletions != 1 || quality[id].ProbeSuccessfulCompletions != 0 || quality[id].ProbeDurationMS != 23 { + t.Fatalf("invalid duration changed probe costs: quality=%+v err=%v", quality[id], err) + } + + // Disabled attribution (source removed from active catalog) does not add + // probe-cost evidence even for a valid lease and contract. + if err := s.SetActiveLearningSources(ctx, nil); err != nil { + t.Fatal(err) + } + if _, err := s.UpsertPrioritizedSourceCounts(ctx, []string{url}, now.Add(3*time.Hour), id); err != nil { + t.Fatal(err) + } + claim, ok, err = s.ClaimDue(ctx, "worker", now.Add(3*time.Hour), time.Minute) + if err != nil || !ok { + t.Fatalf("inactive claim=(%+v,%t,%v)", claim, ok, err) + } + if _, err := s.Complete(ctx, claim, now.Add(3*time.Hour), RetestPolicy{ValidatedAfter: time.Hour, FailedAfter: time.Hour}, true, 200, time.Millisecond, "", OutcomeMeta{ + SourceLearning: true, SourceLearningContract: contractB, SourceProbeDurationMS: 41, + }); err != nil { + t.Fatal(err) + } + quality, err = s.LoadSourceQuality(ctx, []string{id}, now.Add(3*time.Hour)) + if err != nil || quality[id].ProbeCompletions != 1 || quality[id].ProbeDurationMS != 23 { + t.Fatalf("inactive source changed probe costs: quality=%+v err=%v", quality[id], err) + } +} + +func TestMalformedProbeCostCounterDoesNotLoseCommittedOutcome(t *testing.T) { + ctx := context.Background() + mini, s := newTestStore(t, "source-probe-cost-exponent:v1") + now := time.Unix(1_800_000_000, 0) + url := "http://probe-cost-counter.example:8080" + id := StableSourceID("https://feeds.example/probe-cost-counter", "http") + contract := strings.Repeat("a", 64) + if err := s.SetActiveLearningSources(ctx, []string{id}); err != nil { + t.Fatal(err) + } + mini.HSet(s.sourceQualityKey(id), "probe_contract", contract, "probe_completions", "1e3") + if _, err := s.UpsertPrioritizedSourceCounts(ctx, []string{url}, now, id); err != nil { + t.Fatal(err) + } + claim, ok, err := s.ClaimDue(ctx, "worker", now, time.Minute) + if err != nil || !ok { + t.Fatalf("claim=(%+v,%t,%v)", claim, ok, err) + } + result, err := s.Complete(ctx, claim, now, RetestPolicy{ValidatedAfter: time.Hour, FailedAfter: time.Hour}, true, 200, time.Millisecond, "", OutcomeMeta{ + SourceLearning: true, SourceLearningContract: contract, SourceProbeDurationMS: 25, + }) + if err != nil || !result.Committed { + t.Fatalf("malformed cost telemetry prevented committed outcome: result=%+v err=%v", result, err) + } + if got := mini.HGet(s.proxyKey(claim.ID), "last_status"); got != "ok" { + t.Fatalf("candidate outcome was not recorded: last_status=%q", got) + } + if got := mini.HGet(s.proxyKey(claim.ID), "lease_token"); got != "" { + t.Fatalf("candidate lease was not cleared: token=%q", got) + } + if got := mini.HGet(s.sourceQualityKey(id), "probe_completions"); got != "1e3" { + t.Fatalf("malformed probe cost counter changed: %q", got) + } + if got := mini.HGet(s.sourceQualityKey(id), "probe_successful_completions"); got != "" { + t.Fatalf("malformed probe counter wrote success total: %q", got) + } + if got := mini.HGet(s.sourceQualityKey(id), "probe_duration_ms"); got != "" { + t.Fatalf("malformed probe counter wrote duration: %q", got) + } +} + +func TestValidatedDetailsExposeOnlyRoutingEvidenceAndFailClosed(t *testing.T) { + ctx := context.Background() + mini, s := newTestStore(t, "source-routing-evidence:v1") + now := time.Unix(1_800_000_000, 0) + nowMS := now.UnixMilli() + seed := func(id string, values ...string) { + t.Helper() + mini.SAdd(s.validatedKey(), id) + args := append([]string{"ok_ratio_pct", "80"}, values...) + mini.HSet(s.proxyKey(id), args...) + } + seed("fresh", "source_id", StableSourceID("https://feeds.example/new", "http"), "success_streak", "1", "https_ok", "1", "https_checked_at_ms", fmt.Sprint(nowMS), "tampered", "0", "tamper_checked_at_ms", fmt.Sprint(nowMS)) + seed("stale", "source_id", StableSourceID("https://feeds.example/stale", "http"), "success_streak", "3", "https_ok", "1", "https_checked_at_ms", fmt.Sprint(now.Add(-25*time.Hour).UnixMilli()), "tampered", "0", "tamper_checked_at_ms", fmt.Sprint(nowMS)) + seed("future", "source_id", StableSourceID("https://feeds.example/future", "http"), "success_streak", "2", "https_ok", "1", "https_checked_at_ms", fmt.Sprint(now.Add(time.Second).UnixMilli()), "tampered", "0", "tamper_checked_at_ms", fmt.Sprint(nowMS)) + seed("unknown", "source_id", StableSourceID("https://feeds.example/unknown", "http"), "success_streak", "1", "https_ok", "true", "https_checked_at_ms", fmt.Sprint(nowMS), "tampered", "0", "tamper_checked_at_ms", fmt.Sprint(nowMS)) + seed("dirty", "source_id", StableSourceID("https://feeds.example/dirty", "http"), "success_streak", "1", "https_ok", "1", "https_checked_at_ms", fmt.Sprint(nowMS), "tampered", "1", "tamper_checked_at_ms", fmt.Sprint(nowMS)) + infos, err := s.ValidatedDetails(ctx) + if err != nil { + t.Fatal(err) + } + eligible := map[string]bool{} + bySource := map[string]int{} + for _, info := range infos { + eligible[info.SourceID] = info.RoutingEligible(now, 24*time.Hour) + if info.SourceID != "" { + bySource[info.SourceID]++ + } + } + freshSource := StableSourceID("https://feeds.example/new", "http") + if bySource[freshSource] != 1 || !eligible[freshSource] { + t.Fatalf("fresh source evidence count=%d eligible=%v", bySource[freshSource], eligible[freshSource]) + } + for _, id := range []string{"stale", "future", "unknown", "dirty"} { + sourceID := StableSourceID("https://feeds.example/"+id, "http") + if eligible[sourceID] { + t.Fatalf("ineligible evidence fixture %q passed routing gate", id) + } + } + // The method has no last_checked_at gate; its API is exactly the listed + // evidence tuple plus ratio and streak. + noLastCheck := ValidatedInfo{OkRatioPct: 80, SuccessStreak: ptr(int64(1)), HTTPSOK: ptr(true), HTTPSCheckedAt: nowMS, Tampered: ptr(false), TamperCheckedAt: nowMS} + if !noLastCheck.RoutingEligible(now, 24*time.Hour) { + t.Fatal("routing eligibility added an unrelated last-check timestamp gate") + } + if noLastCheck.RoutingEligible(now, time.Hour) == false { + t.Fatal("fresh routing evidence should remain eligible with a stricter age bound") + } + missingTimes := noLastCheck + missingTimes.HTTPSCheckedAt, missingTimes.TamperCheckedAt = 0, 0 + if missingTimes.RoutingEligible(now, 0) { + t.Fatal("zero max age must not trust missing check timestamps") + } +} + +func ptr[T any](value T) *T { return &value } diff --git a/internal/store/source_learning.go b/internal/store/source_learning.go index 56b86de..39c60c6 100644 --- a/internal/store/source_learning.go +++ b/internal/store/source_learning.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "math" + "sort" "strconv" "strings" "time" @@ -18,6 +19,10 @@ const ( sourceQualityTTL = 30 * 24 * time.Hour sourceFailureBaseCooldown = time.Minute sourceFailureMaxCooldown = time.Hour + maxSourceHTTPAttempts = int64(1000) + maxSourceNetworkBytes = int64(1 << 30) + maxSourceCostDurationMS = int64(24 * time.Hour / time.Millisecond) + maxSourceAdmissionBatch = int64(1_000_000_000) sourceStageScoreStride = int64(1 << 16) maxSourceStagePriority = int64((1<<53)-1) / sourceStageScoreStride MaxSourceQualityReadBatch = 512 @@ -30,24 +35,128 @@ var ErrSourceLearningBatchTooLarge = errors.New("source quality read exceeds bat // stored fields are zeroed and reported through Corrupt rather than failing a // whole refresh-plan read. type SourceQuality struct { - RefreshFetches int64 - RefreshSuccesses int64 - FetchFailures int64 - ConsecutiveFailures int64 - Parsed int64 - UniqueIncremental int64 - DuplicateRecords int64 - LastFetchAt time.Time - LastSuccessAt time.Time - CooldownUntil time.Time - ProbeSeen int64 - ProbeSamples float64 - ProbePasses float64 - ProbeUpdatedAt time.Time - ProbeContract string - Corrupt bool + RefreshFetches int64 + RefreshSuccesses int64 + FetchFailures int64 + ConsecutiveFailures int64 + Parsed int64 + UniqueIncremental int64 + DuplicateRecords int64 + LastFetchAt time.Time + LastSuccessAt time.Time + CooldownUntil time.Time + Admitted int64 + AdmissionRefreshes int64 + LastAdmitted int64 + ZeroAdmissionStreak int64 + LastAdmissionAt time.Time + HTTPAttempts int64 + NetworkBytes int64 + FetchDurationMS int64 + FetchCostObservations int64 + ProbeSeen int64 + ProbeSamples float64 + ProbePasses float64 + ProbeCompletions int64 + ProbeSuccessfulCompletions int64 + ProbeDurationMS int64 + ProbeUpdatedAt time.Time + ProbeContract string + Corrupt bool } +// RecordSourceFetchCost adds bounded request-cost observations for one source +// refresh attempt sequence. The source hash retention is refreshed atomically. +func (s *Redis) RecordSourceFetchCost(ctx context.Context, sourceID string, httpAttempts, networkBytes, durationMS int64, now time.Time) error { + if !validSourceID(sourceID) || now.UnixMilli() <= 0 || + httpAttempts < 0 || httpAttempts > maxSourceHTTPAttempts || + networkBytes < 0 || networkBytes > maxSourceNetworkBytes || + durationMS < 0 || durationMS > maxSourceCostDurationMS { + return fmt.Errorf("invalid source fetch cost") + } + return recordSourceFetchCostScript.Run(ctx, s.client, []string{s.sourceQualityKey(sourceID)}, + httpAttempts, networkBytes, durationMS, sourceQualityTTL.Milliseconds()).Err() +} + +var recordSourceFetchCostScript = redis.NewScript(` +local key = KEYS[1] +local fields = {'http_attempts', 'network_bytes', 'fetch_duration_ms', 'fetch_cost_observations'} +local increments = {tonumber(ARGV[1]), tonumber(ARGV[2]), tonumber(ARGV[3]), 1} +local cap = 9007199254740991 +for i = 1, #fields do + local raw = redis.call('HGET', key, fields[i]) or '0' + local current = tonumber(raw) + if not string.match(raw, '^%d+$') or not current or current < 0 or current ~= math.floor(current) or current + increments[i] > cap then + return redis.error_reply('invalid source cost counter') + end +end +for i = 1, #fields do redis.call('HINCRBY', key, fields[i], tostring(increments[i])) end +redis.call('PEXPIRE', key, ARGV[4]) +return 1 +`) + +// RecordSourceAdmissionRefresh records the per-source admission result from +// one completed global refresh. Callers must include successful participating +// sources with zero admissions; failed or skipped feeds are omitted. Admitted +// is counted only by the upsert SADD script, so partial refreshes retain their +// actual insertions without being mistaken for completed observations. +func (s *Redis) RecordSourceAdmissionRefresh(ctx context.Context, admittedBySource map[string]int64, now time.Time) error { + if len(admittedBySource) > MaxSourceQualityReadBatch || now.UnixMilli() <= 0 { + return fmt.Errorf("invalid source admission refresh") + } + if len(admittedBySource) == 0 { + return nil + } + ids := make([]string, 0, len(admittedBySource)) + for id, admitted := range admittedBySource { + if !validSourceID(id) || admitted < 0 || admitted > maxSourceAdmissionBatch { + return fmt.Errorf("invalid source admission refresh") + } + ids = append(ids, id) + } + sort.Strings(ids) + keys := make([]string, len(ids)) + args := make([]interface{}, 0, len(ids)+2) + args = append(args, now.UnixMilli(), sourceQualityTTL.Milliseconds()) + for i, id := range ids { + keys[i] = s.sourceQualityKey(id) + args = append(args, admittedBySource[id]) + } + return recordSourceAdmissionRefreshScript.Run(ctx, s.client, keys, args...).Err() +} + +var recordSourceAdmissionRefreshScript = redis.NewScript(` +local cap = 9007199254740991 +for i = 1, #KEYS do + local admitted = tonumber(ARGV[i + 2]) + if not admitted or admitted < 0 or admitted ~= math.floor(admitted) or admitted > 1000000000 then + return redis.error_reply('invalid source admission count') + end + local fields = {'admission_refreshes', 'zero_admission_streak'} + local increments = {1, admitted == 0 and 1 or 0} + for j = 1, #fields do + local raw = redis.call('HGET', KEYS[i], fields[j]) or '0' + local current = tonumber(raw) + if not string.match(raw, '^%d+$') or not current or current < 0 or current ~= math.floor(current) or current + increments[j] > cap then + return redis.error_reply('invalid source admission counter') + end + end +end +for i = 1, #KEYS do + local key = KEYS[i] + local admitted = tonumber(ARGV[i + 2]) + redis.call('HINCRBY', key, 'admission_refreshes', 1) + if admitted == 0 then + redis.call('HINCRBY', key, 'zero_admission_streak', '1') + else + redis.call('HSET', key, 'zero_admission_streak', '0') + end + redis.call('HSET', key, 'last_admitted', ARGV[i + 2], 'last_admission_ms', ARGV[1]) + redis.call('PEXPIRE', key, ARGV[2]) +end +return 1 +`) + // StableSourceID identifies the exact trimmed feed URL and its parser scheme // hint. Equivalent URLs with different query strings stay distinct, as do // feeds whose bare-address rows use different schemes. @@ -266,9 +375,21 @@ func decodeSourceQuality(values map[string]string, now time.Time) SourceQuality q.LastFetchAt = timeField("last_fetch_ms") q.LastSuccessAt = timeField("last_success_ms") q.CooldownUntil = timeField("cooldown_until_ms") + q.Admitted = intField("admitted") + q.AdmissionRefreshes = intField("admission_refreshes") + q.LastAdmitted = intField("last_admitted") + q.ZeroAdmissionStreak = intField("zero_admission_streak") + q.LastAdmissionAt = timeField("last_admission_ms") + q.HTTPAttempts = intField("http_attempts") + q.NetworkBytes = intField("network_bytes") + q.FetchDurationMS = intField("fetch_duration_ms") + q.FetchCostObservations = intField("fetch_cost_observations") q.ProbeSeen = intField("probe_seen") q.ProbeSamples = floatField("probe_samples") q.ProbePasses = floatField("probe_passes") + q.ProbeCompletions = intField("probe_completions") + q.ProbeSuccessfulCompletions = intField("probe_successful_completions") + q.ProbeDurationMS = intField("probe_duration_ms") q.ProbeUpdatedAt = timeField("probe_updated_ms") if contract := values["probe_contract"]; contract != "" { if validProbeContract(contract) { @@ -300,10 +421,11 @@ func validProbeContract(contract string) bool { // check in completeScript, before any early discard/delete branch. The caller // adds SourceQualityKeyPrefix as KEYS[6], then appends enabled, // sampleEvery, halfLifeMS, and the optional 64-hex probe contract as -// ARGV[28..32]. A contract change resets only sampled probe outcomes before -// recording the first new-contract result; fetch and refresh counters remain. -// It samples each source's first completed probe and every Nth retest. -// The body is constant-time and adds no Redis round trip per probe. +// ARGV[28..32], followed by probe duration at ARGV[33]. A contract change +// resets sampled outcomes and committed probe-cost counters before recording +// the first new-contract result; fetch and refresh counters remain. It samples +// each source's first completed probe and every Nth retest, while cost counters +// include every committed probe. The body adds no Redis round trip per probe. const SourceProbeLearningLuaSnippet = ` if ARGV[28] == '1' then local source_id = redis.call('HGET', key, 'source_id') @@ -320,7 +442,9 @@ if source_id and string.len(source_id) == 32 and string.match(source_id, '^[0-9a if previous_contract ~= contract then redis.call('HSET', quality_key, 'probe_seen', '0', 'probe_samples', '0', 'probe_passes', '0', - 'probe_updated_ms', '0', 'probe_contract', contract) + 'probe_updated_ms', '0', 'probe_completions', '0', + 'probe_successful_completions', '0', 'probe_duration_ms', '0', + 'probe_contract', contract) contract_changed = true end end @@ -344,10 +468,42 @@ if source_id and string.len(source_id) == 32 and string.match(source_id, '^[0-9a 'probe_updated_ms', now_ms) redis.call('PEXPIRE', quality_key, 2592000000) end + local probe_duration_ms = tonumber(ARGV[33]) + if string.len(contract) == 64 and string.match(contract, '^[0-9a-f]+$') and + probe_duration_ms and probe_duration_ms >= 0 and probe_duration_ms <= 86400000 then + local success_increment = status == 'ok' and 1 or 0 + local cap = 9007199254740991 + local completion_raw = redis.call('HGET', quality_key, 'probe_completions') or '0' + local successful_raw = redis.call('HGET', quality_key, 'probe_successful_completions') or '0' + local duration_raw = redis.call('HGET', quality_key, 'probe_duration_ms') or '0' + local completion_count = tonumber(completion_raw) + local successful_count = tonumber(successful_raw) + local duration_total = tonumber(duration_raw) + local valid_cost = string.match(completion_raw, '^%d+$') and + string.match(successful_raw, '^%d+$') and string.match(duration_raw, '^%d+$') and + completion_count and successful_count and duration_total and + completion_count >= 0 and successful_count >= 0 and duration_total >= 0 and + completion_count == math.floor(completion_count) and successful_count == math.floor(successful_count) and + duration_total == math.floor(duration_total) and completion_count + 1 <= cap and + successful_count + success_increment <= cap and duration_total + probe_duration_ms <= cap + if valid_cost then + redis.call('HINCRBY', quality_key, 'probe_completions', '1') + if success_increment == 1 then redis.call('HINCRBY', quality_key, 'probe_successful_completions', '1') end + redis.call('HINCRBY', quality_key, 'probe_duration_ms', tostring(probe_duration_ms)) + redis.call('PEXPIRE', quality_key, '2592000000') + end + end end end ` +func sourceProbeDurationArg(durationMS int64) int64 { + if durationMS < 0 || durationMS > maxSourceCostDurationMS { + return -1 + } + return durationMS +} + // DecayedSourceYield applies the configured half-life at read time. Invalid or // absent values naturally produce the Beta(1,1) prior of 0.5. func DecayedSourceYield(q SourceQuality, now time.Time, halfLife time.Duration) (yield, effectiveSamples float64) { diff --git a/k8s/overlays/live-local/monitor.json b/k8s/overlays/live-local/monitor.json index 31aad44..aa987d1 100644 --- a/k8s/overlays/live-local/monitor.json +++ b/k8s/overlays/live-local/monitor.json @@ -943,6 +943,8 @@ "control_probe_failure_threshold": 3, "classification_max_age": "24h", "source_learning_enabled": true, + "source_learning_zero_admission_cooldown": "1h", + "source_learning_zero_admission_refreshes": 3, "target_profiles": [ { "name": "crawlora-content", From cb423177a0a5da574b8bc0dae82ffa1bf6ac02a1 Mon Sep 17 00:00:00 2001 From: tonywangcn Date: Wed, 14 Oct 2026 01:34:00 +0800 Subject: [PATCH 2/2] Keep malformed telemetry and static checks safe --- cmd/freeproxyapi/feed_audit.go | 5 ----- .../source_efficiency_integration_test.go | 3 +++ internal/store/redis.go | 2 +- internal/store/source_efficiency_test.go | 18 ++++++++++++++++++ internal/store/source_learning.go | 12 ++++++++---- 5 files changed, 30 insertions(+), 10 deletions(-) diff --git a/cmd/freeproxyapi/feed_audit.go b/cmd/freeproxyapi/feed_audit.go index 3add78f..181accf 100644 --- a/cmd/freeproxyapi/feed_audit.go +++ b/cmd/freeproxyapi/feed_audit.go @@ -68,11 +68,6 @@ type feedAuditOptions struct { MaxRecords int } -type feedAuditFlags struct { - inputPath string - options feedAuditOptions -} - func feedAuditCommand(args []string, out io.Writer) error { flags := flag.NewFlagSet("feed-audit", flag.ContinueOnError) flags.SetOutput(io.Discard) diff --git a/internal/monitor/source_efficiency_integration_test.go b/internal/monitor/source_efficiency_integration_test.go index 2067d13..70336be 100644 --- a/internal/monitor/source_efficiency_integration_test.go +++ b/internal/monitor/source_efficiency_integration_test.go @@ -61,6 +61,9 @@ func TestSourceAdmissionCooldownUsesStoredFullRefreshObservations(t *testing.T) t.Fatal(err) } quality, err = runner.store.LoadSourceQuality(ctx, []string{id}, now.Add(5*time.Minute)) + if err != nil || quality[id].ZeroAdmissionStreak != 1 { + t.Fatalf("zero streak after reset = %+v, err=%v", quality[id], err) + } if err != nil { t.Fatal(err) } diff --git a/internal/store/redis.go b/internal/store/redis.go index 2e020e4..ce3c03c 100644 --- a/internal/store/redis.go +++ b/internal/store/redis.go @@ -412,7 +412,7 @@ if source_id then if current_raw == false then current_raw = '0' end local current = tonumber(current_raw) local max_batch = math.floor((last_pair - 1) / 2) - if current and string.match(current_raw, '^%d+$') and current >= 0 and + if current and (current_raw == '0' or string.match(current_raw, '^[1-9]%d*$')) and current >= 0 and current == math.floor(current) and current + max_batch <= 9007199254740991 then record_admission = true end diff --git a/internal/store/source_efficiency_test.go b/internal/store/source_efficiency_test.go index 761cc64..f6820b5 100644 --- a/internal/store/source_efficiency_test.go +++ b/internal/store/source_efficiency_test.go @@ -458,3 +458,21 @@ func TestValidatedDetailsExposeOnlyRoutingEvidenceAndFailClosed(t *testing.T) { } func ptr[T any](value T) *T { return &value } + +func TestNonCanonicalLeadingZeroTelemetryIsRejectedBeforeMutation(t *testing.T) { + ctx := context.Background() + mini, s := newTestStore(t, "counter-canonical:v1") + now := time.Unix(1800000000, 0) + id := StableSourceID("https://feeds.example/canonical", "http") + mini.HSet(s.sourceQualityKey(id), "http_attempts", "7", "network_bytes", "01") + if err := s.RecordSourceFetchCost(ctx, id, 1, 2, 3, now); err == nil { + t.Fatal("leading-zero counter accepted") + } + if mini.HGet(s.sourceQualityKey(id), "http_attempts") != "7" { + t.Fatal("preflight partially updated telemetry") + } + quality, err := s.LoadSourceQuality(ctx, []string{id}, now) + if err != nil || !quality[id].Corrupt { + t.Fatal("noncanonical counter was not marked corrupt") + } +} diff --git a/internal/store/source_learning.go b/internal/store/source_learning.go index 39c60c6..4860317 100644 --- a/internal/store/source_learning.go +++ b/internal/store/source_learning.go @@ -86,7 +86,7 @@ local cap = 9007199254740991 for i = 1, #fields do local raw = redis.call('HGET', key, fields[i]) or '0' local current = tonumber(raw) - if not string.match(raw, '^%d+$') or not current or current < 0 or current ~= math.floor(current) or current + increments[i] > cap then + if not (raw == '0' or string.match(raw, '^[1-9]%d*$')) or not current or current < 0 or current ~= math.floor(current) or current + increments[i] > cap then return redis.error_reply('invalid source cost counter') end end @@ -137,7 +137,7 @@ for i = 1, #KEYS do for j = 1, #fields do local raw = redis.call('HGET', KEYS[i], fields[j]) or '0' local current = tonumber(raw) - if not string.match(raw, '^%d+$') or not current or current < 0 or current ~= math.floor(current) or current + increments[j] > cap then + if not (raw == '0' or string.match(raw, '^[1-9]%d*$')) or not current or current < 0 or current ~= math.floor(current) or current + increments[j] > cap then return redis.error_reply('invalid source admission counter') end end @@ -339,6 +339,10 @@ func decodeSourceQuality(values map[string]string, now time.Time) SourceQuality if !ok || raw == "" { return 0 } + if raw != "0" && (strings.HasPrefix(raw, "0") || strings.HasPrefix(raw, "+")) { + q.Corrupt = true + return 0 + } v, err := strconv.ParseInt(raw, 10, 64) if err != nil || v < 0 { q.Corrupt = true @@ -479,8 +483,8 @@ if source_id and string.len(source_id) == 32 and string.match(source_id, '^[0-9a local completion_count = tonumber(completion_raw) local successful_count = tonumber(successful_raw) local duration_total = tonumber(duration_raw) - local valid_cost = string.match(completion_raw, '^%d+$') and - string.match(successful_raw, '^%d+$') and string.match(duration_raw, '^%d+$') and + local valid_cost = (completion_raw == '0' or string.match(completion_raw, '^[1-9]%d*$')) and + (successful_raw == '0' or string.match(successful_raw, '^[1-9]%d*$')) and (duration_raw == '0' or string.match(duration_raw, '^[1-9]%d*$')) and completion_count and successful_count and duration_total and completion_count >= 0 and successful_count >= 0 and duration_total >= 0 and completion_count == math.floor(completion_count) and successful_count == math.floor(successful_count) and