Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions database/analytics_filters.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ func buildHitFilter(filterType, filterValue, alias string) (string, []any) {
if isDirectReferrer(filterValue) {
normalized = "(Direct)"
}
expr := fmt.Sprintf("hk_referrer(%sreferrer)", prefix)
expr := fmt.Sprintf("hk_external_referrer(%sreferrer, %ssite_id)", prefix, prefix)
return " AND " + expr + " = ?", []any{normalized}
case "referrer_host":
expr := referrerHostExpr(prefix)
Expand Down Expand Up @@ -162,8 +162,9 @@ func normalizeReferrerHostFilter(value string) string {

func referrerHostExpr(prefix string) string {
return fmt.Sprintf(`CASE
WHEN hk_external_referrer(%sreferrer, %ssite_id) IS NULL THEN NULL
WHEN %sreferrer IS NULL OR NULLIF(TRIM(%sreferrer), '') IS NULL THEN '(Direct)'
WHEN lower(%sreferrer) LIKE 'http%%' THEN regexp_replace(regexp_extract(lower(%sreferrer), 'https?://([^/:?#]+)', 1), '^www\\.', '')
ELSE regexp_replace(lower(TRIM(%sreferrer)), '^www\\.', '')
END`, prefix, prefix, prefix, prefix, prefix)
END`, prefix, prefix, prefix, prefix, prefix, prefix, prefix)
}
1 change: 1 addition & 0 deletions database/analytics_filters_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ func TestBuildHitFilter(t *testing.T) {
field: "referrer_host",
value: "www.Google.com",
wantClause: ` AND CASE
WHEN hk_external_referrer(h.referrer, h.site_id) IS NULL THEN NULL
WHEN h.referrer IS NULL OR NULLIF(TRIM(h.referrer), '') IS NULL THEN '(Direct)'
WHEN lower(h.referrer) LIKE 'http%' THEN regexp_replace(regexp_extract(lower(h.referrer), 'https?://([^/:?#]+)', 1), '^www\\.', '')
ELSE regexp_replace(lower(TRIM(h.referrer)), '^www\\.', '')
Expand Down
16 changes: 16 additions & 0 deletions database/migrations/2026_11_16_000000_normalize_self_referrers.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
-- Classify internal navigation at query time, including stored/imported history.
CREATE OR REPLACE MACRO hk_external_referrer(ref, ref_site_id) AS (
CASE
WHEN regexp_replace(
regexp_extract(lower(trim(ref)), '^(?:https?://|//)?([^/:?#]+)(?::[0-9]+)?(?:[/?#]|$)', 1),
'^www\.', ''
) = (
SELECT regexp_replace(lower(trim(domain)), '^www\.', '')
FROM sites WHERE id = ref_site_id
) THEN NULL
-- Inline hk_referrer's fallback: IMPORT may restore this macro first.
WHEN ref IS NULL OR ref = '' THEN '(Direct)'
WHEN ref LIKE 'http%' THEN regexp_extract(ref, 'https?://([^/]+)', 1)
ELSE ref
END
);
16 changes: 16 additions & 0 deletions database/migrations/tenant/0018_normalize_self_referrers.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
-- Classify internal navigation at query time, including stored/imported history.
CREATE OR REPLACE MACRO hk_external_referrer(ref, ref_site_id) AS (
CASE
WHEN regexp_replace(
regexp_extract(lower(trim(ref)), '^(?:https?://|//)?([^/:?#]+)(?::[0-9]+)?(?:[/?#]|$)', 1),
'^www\.', ''
) = (
SELECT regexp_replace(lower(trim(domain)), '^www\.', '')
FROM sites WHERE id = ref_site_id
) THEN NULL
-- Inline hk_referrer's fallback: IMPORT may restore this macro first.
WHEN ref IS NULL OR ref = '' THEN '(Direct)'
WHEN ref LIKE 'http%' THEN regexp_extract(ref, 'https?://([^/]+)', 1)
ELSE ref
END
);
2 changes: 1 addition & 1 deletion database/store_analytics_breakdown.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import (
// using the same classifications as the site overview's top lists.
var breakdownDimensions = map[string]string{
"page": "h.path",
"referrer": "hk_referrer(h.referrer)",
"referrer": "hk_external_referrer(h.referrer, h.site_id)",
"device": "hk_device(h.viewport_width)",
"country": "hk_country(h.country_code)",
"city": "COALESCE(NULLIF(TRIM(h.city), ''), '(Unknown)')",
Expand Down
4 changes: 3 additions & 1 deletion database/store_analytics_events.go
Original file line number Diff line number Diff line change
Expand Up @@ -247,7 +247,7 @@ func (s *Store) GetEventAudience(ctx context.Context, params api.EventAudiencePa
base AS (
SELECT
h.path AS path,
hk_referrer(h.referrer) AS referrer,
hk_external_referrer(h.referrer, h.site_id) AS referrer,
hk_device(h.viewport_width) AS device,
hk_country(h.country_code) AS country,
COALESCE(NULLIF(TRIM(h.city), ''), '(Unknown)') AS city,
Expand Down Expand Up @@ -278,6 +278,7 @@ func (s *Store) GetEventAudience(ctx context.Context, params api.EventAudiencePa
SELECT dim, name, val,
ROW_NUMBER() OVER (PARTITION BY dim ORDER BY val DESC) AS rn
FROM agg
WHERE name IS NOT NULL
)
SELECT dim, name, val FROM ranked WHERE rn <= 10 ORDER BY dim, val DESC
`, propClause, filterSQL)
Expand Down Expand Up @@ -424,6 +425,7 @@ func (s *Store) getImportedEventAudienceDimensions(ctx context.Context, params a
FROM imported_event_dimensions_daily
WHERE site_id = ? AND date >= CAST(? AS DATE) AND date <= CAST(? AS DATE)
AND event_name = ? AND dimension IN ('path', 'referrer', 'device', 'country', 'city', 'provider', 'asn')
AND (dimension <> 'referrer' OR hk_external_referrer(name, site_id) IS NOT NULL)
GROUP BY dimension, name
UNION ALL
SELECT 'path' AS dimension, path AS name, SUM(visitors) AS val
Expand Down
4 changes: 2 additions & 2 deletions database/store_analytics_filters.go
Original file line number Diff line number Diff line change
Expand Up @@ -287,9 +287,9 @@ const agentsCTE = `agents AS (
const referrersCTE = `referrers AS (
SELECT
referrer,
hk_referrer(referrer) AS referrer_name,
hk_external_referrer(referrer, site_id) AS referrer_name,
hk_ai_source(referrer) AS ai_source
FROM (SELECT DISTINCT referrer FROM scoped)
FROM (SELECT DISTINCT referrer, site_id FROM scoped)
)`

// queryAIKpis aggregates the AI traffic KPIs for one window. It mirrors the
Expand Down
6 changes: 4 additions & 2 deletions database/store_analytics_imported.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"context"
"database/sql"
"fmt"
"math"
"slices"
"time"

Expand Down Expand Up @@ -47,7 +48,7 @@ func (s *Store) augmentImportedSiteStats(ctx context.Context, params api.Analyti
importedDuration := float64(sqlNullInt64Value(duration))
nativePageviews := stats.TotalPageviews
nativeSessions := stats.UniqueSessions
nativeBounces := int((stats.BounceRate / 100) * float64(nativeSessions))
nativeBounces := int(math.Round((stats.BounceRate / 100) * float64(nativeSessions)))
nativeDuration := stats.AvgSessionDuration * float64(nativeSessions)

stats.TotalPageviews += importedPageviews
Expand Down Expand Up @@ -96,7 +97,7 @@ func (s *Store) augmentImportedSiteOverviewStats(ctx context.Context, params api
importedVisits := int(sqlNullInt64Value(visits))
importedBounces := int(sqlNullInt64Value(bounces))
nativeSessions := stats.UniqueSessions
nativeBounces := int((stats.BounceRate / 100) * float64(nativeSessions))
nativeBounces := int(math.Round((stats.BounceRate / 100) * float64(nativeSessions)))

stats.TotalPageviews += importedPageviews
stats.UniqueSessions += importedVisits
Expand Down Expand Up @@ -249,6 +250,7 @@ func (s *Store) mergeImportedTopLists(ctx context.Context, params api.AnalyticsP
SELECT dimension, name, SUM(pageviews) AS val
FROM imported_dimension_daily
WHERE site_id = ? AND date >= CAST(? AS DATE) AND date <= CAST(? AS DATE)
AND (dimension <> 'source' OR hk_external_referrer(name, site_id) IS NOT NULL)
AND dimension IN (
'page', 'entry_page', 'exit_page', 'source', 'device', 'country', 'browser', 'operating_system',
'city', 'provider', 'asn', 'language', 'utm_campaign', 'utm_content', 'utm_medium', 'utm_source', 'utm_term'
Expand Down
66 changes: 66 additions & 0 deletions database/store_analytics_imported_bounce_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
package database

import (
"math"
"testing"
"time"

"github.com/google/uuid"

"hitkeep/api"
"hitkeep/importables"
)

func TestImportedStatsPreserveNativeBounceCount(t *testing.T) {
store, userID := setupEventBreakdownStore(t)
ctx := t.Context()
site, err := store.CreateSite(ctx, userID, "bounce.example.test")
if err != nil {
t.Fatal(err)
}
at := time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC)
for _, duration := range []time.Duration{0, 20 * time.Second, 40 * time.Second} {
session := uuid.New()
if err := store.CreateHit(ctx, &api.Hit{SiteID: site.ID, SessionID: session, PageID: uuid.New(), Timestamp: at, Path: "/first"}); err != nil {
t.Fatal(err)
}
if duration > 0 {
if err := store.CreateHit(ctx, &api.Hit{SiteID: site.ID, SessionID: session, PageID: uuid.New(), Timestamp: at.Add(duration), Path: "/second"}); err != nil {
t.Fatal(err)
}
}
}
params := api.AnalyticsParams{SiteID: site.ID, Start: at.AddDate(0, 0, -1), End: at.AddDate(0, 0, 1)}
for _, imported := range []bool{false, true} {
if imported {
sink, err := NewImportedDataSink(ctx, store, site.ID, uuid.New())
if err != nil {
t.Fatal(err)
}
if err := sink.PutTraffic(ctx, importables.TrafficRow{Date: at.AddDate(0, 0, -1), Visits: 1, Visitors: 1, Pageviews: 1, Bounces: 1, VisitDuration: 120, SourceFile: "visitors.csv"}); err != nil {
t.Fatal(err)
}
if err := sink.Flush(ctx); err != nil {
t.Fatal(err)
}
}
wantBounce, wantDuration := 100.0/3, 20.0
if imported {
wantBounce, wantDuration = 50, 45
}
stats, err := store.GetSiteStats(ctx, params)
if err != nil {
t.Fatal(err)
}
if math.Abs(stats.BounceRate-wantBounce) > 1e-9 || stats.AvgSessionDuration != wantDuration {
t.Errorf("imported=%t: bounce=%v duration=%v, want %v and %v", imported, stats.BounceRate, stats.AvgSessionDuration, wantBounce, wantDuration)
}
overview, err := store.GetSiteOverviewStats(ctx, params)
if err != nil {
t.Fatal(err)
}
if math.Abs(overview.BounceRate-wantBounce) > 1e-9 {
t.Errorf("imported=%t overview bounce=%v, want %v", imported, overview.BounceRate, wantBounce)
}
}
}
2 changes: 1 addition & 1 deletion database/store_ecommerce.go
Original file line number Diff line number Diff line change
Expand Up @@ -402,7 +402,7 @@ func (s *Store) listEcommerceEvents(ctx context.Context, params api.EcommercePar
arg_min(COALESCE(NULLIF(TRIM(h.utm_source), ''), '(Unspecified)'), h.timestamp) AS utm_source,
arg_min(COALESCE(NULLIF(TRIM(h.utm_medium), ''), '(Unspecified)'), h.timestamp) AS utm_medium,
arg_min(COALESCE(NULLIF(TRIM(h.utm_campaign), ''), '(Unspecified)'), h.timestamp) AS utm_campaign,
arg_min(hk_referrer(h.referrer), h.timestamp) AS referrer,
arg_min(hk_external_referrer(h.referrer, h.site_id), h.timestamp) AS referrer,
arg_min(hk_device(h.viewport_width), h.timestamp) AS device,
arg_min(hk_country(h.country_code), h.timestamp) AS country,
arg_min(COALESCE(NULLIF(TRIM(h.city), ''), '(Unknown)'), h.timestamp) AS city,
Expand Down
Loading
Loading