Files
cburn/internal/pipeline/aggregator.go
teernisse 24454247a3 feat: add data pipeline with parallel loading, aggregation, and cache integration
Implement the pipeline layer that orchestrates discovery, parsing,
caching, and aggregation:

- pipeline/loader.go: Load() discovers session files via ScanDir,
  optionally filters out subagent files, then parses all files in
  parallel using a bounded worker pool sized to GOMAXPROCS. Workers
  read from a pre-filled channel (no contention on dispatch) and
  report progress via an atomic counter and callback. LoadResult
  tracks total files, parsed files, parse errors, and file errors.

- pipeline/aggregator.go: Five aggregation functions, all operating
  on time-filtered session slices:

  * Aggregate: computes SummaryStats across all sessions — total
    tokens (5 types), estimated cost, cache savings (summed per-model
    via config.CalculateCacheSavings), cache hit rate, and per-active-
    day rates (cost, tokens, sessions, prompts, minutes).

  * AggregateDays: groups sessions by local calendar date, sorted
    most-recent-first.

  * AggregateModels: groups by normalized model name with share
    percentages, sorted by cost descending.

  * AggregateProjects: groups by project name, sorted by cost.

  * AggregateHourly: distributes prompt/session/token counts across
    24 hour buckets (attributed to session start hour).

  Also provides FilterByTime, FilterByProject, FilterByModel with
  case-insensitive substring matching.

- pipeline/incremental.go: LoadWithCache() implements the incremental
  loading strategy — compares discovered files against the cache's
  file_tracker (mtime_ns + size), loads unchanged sessions from
  SQLite, and only reparses files that changed. Reparsed results
  are immediately saved back to cache. CacheDir/CachePath follow
  XDG_CACHE_HOME convention (~/.cache/cburn/metrics.db).

- pipeline/bench_test.go: Benchmarks for ScanDir, ParseFile (worst-
  case largest file), full Load, and LoadWithCache to measure the
  incremental cache speedup.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-19 13:01:56 -05:00

262 lines
7.2 KiB
Go

package pipeline
import (
"sort"
"strings"
"time"
"cburn/internal/config"
"cburn/internal/model"
)
// Aggregate computes summary statistics from a slice of session stats,
// filtered to sessions within the given time range.
func Aggregate(sessions []model.SessionStats, since, until time.Time) model.SummaryStats {
filtered := FilterByTime(sessions, since, until)
var stats model.SummaryStats
activeDays := make(map[string]struct{})
for _, s := range filtered {
stats.TotalSessions++
stats.TotalPrompts += s.UserMessages
stats.TotalAPICalls += s.APICalls
stats.TotalDurationSecs += s.DurationSecs
stats.InputTokens += s.InputTokens
stats.OutputTokens += s.OutputTokens
stats.CacheCreation5mTokens += s.CacheCreation5mTokens
stats.CacheCreation1hTokens += s.CacheCreation1hTokens
stats.CacheReadTokens += s.CacheReadTokens
stats.EstimatedCost += s.EstimatedCost
if !s.StartTime.IsZero() {
day := s.StartTime.Local().Format("2006-01-02")
activeDays[day] = struct{}{}
}
}
stats.ActiveDays = len(activeDays)
stats.TotalBilledTokens = stats.InputTokens + stats.OutputTokens +
stats.CacheCreation5mTokens + stats.CacheCreation1hTokens
// Cache hit rate
totalCacheInput := stats.CacheReadTokens + stats.CacheCreation5mTokens +
stats.CacheCreation1hTokens + stats.InputTokens
if totalCacheInput > 0 {
stats.CacheHitRate = float64(stats.CacheReadTokens) / float64(totalCacheInput)
}
// Cache savings (sum across all models found in sessions)
for _, s := range filtered {
for modelName, mu := range s.Models {
stats.CacheSavings += config.CalculateCacheSavings(modelName, mu.CacheReadTokens)
}
}
// Per-active-day rates
if stats.ActiveDays > 0 {
days := float64(stats.ActiveDays)
stats.CostPerDay = stats.EstimatedCost / days
stats.TokensPerDay = int64(float64(stats.TotalBilledTokens) / days)
stats.SessionsPerDay = float64(stats.TotalSessions) / days
stats.PromptsPerDay = float64(stats.TotalPrompts) / days
stats.MinutesPerDay = float64(stats.TotalDurationSecs) / 60 / days
}
return stats
}
// AggregateDays computes per-day statistics from sessions.
func AggregateDays(sessions []model.SessionStats, since, until time.Time) []model.DailyStats {
filtered := FilterByTime(sessions, since, until)
dayMap := make(map[string]*model.DailyStats)
for _, s := range filtered {
if s.StartTime.IsZero() {
continue
}
dayKey := s.StartTime.Local().Format("2006-01-02")
ds, ok := dayMap[dayKey]
if !ok {
t, _ := time.ParseInLocation("2006-01-02", dayKey, time.Local)
ds = &model.DailyStats{Date: t}
dayMap[dayKey] = ds
}
ds.Sessions++
ds.Prompts += s.UserMessages
ds.APICalls += s.APICalls
ds.DurationSecs += s.DurationSecs
ds.InputTokens += s.InputTokens
ds.OutputTokens += s.OutputTokens
ds.CacheCreation5m += s.CacheCreation5mTokens
ds.CacheCreation1h += s.CacheCreation1hTokens
ds.CacheReadTokens += s.CacheReadTokens
ds.EstimatedCost += s.EstimatedCost
}
// Convert to sorted slice (most recent first)
days := make([]model.DailyStats, 0, len(dayMap))
for _, ds := range dayMap {
days = append(days, *ds)
}
sort.Slice(days, func(i, j int) bool {
return days[i].Date.After(days[j].Date)
})
return days
}
// AggregateModels computes per-model statistics from sessions.
func AggregateModels(sessions []model.SessionStats, since, until time.Time) []model.ModelStats {
filtered := FilterByTime(sessions, since, until)
modelMap := make(map[string]*model.ModelStats)
totalCalls := 0
for _, s := range filtered {
for modelName, mu := range s.Models {
ms, ok := modelMap[modelName]
if !ok {
ms = &model.ModelStats{Model: modelName}
modelMap[modelName] = ms
}
ms.APICalls += mu.APICalls
ms.InputTokens += mu.InputTokens
ms.OutputTokens += mu.OutputTokens
ms.CacheCreation5m += mu.CacheCreation5mTokens
ms.CacheCreation1h += mu.CacheCreation1hTokens
ms.CacheReadTokens += mu.CacheReadTokens
ms.EstimatedCost += mu.EstimatedCost
totalCalls += mu.APICalls
}
}
// Compute share percentages and sort by cost descending
models := make([]model.ModelStats, 0, len(modelMap))
for _, ms := range modelMap {
if totalCalls > 0 {
ms.SharePercent = float64(ms.APICalls) / float64(totalCalls) * 100
}
models = append(models, *ms)
}
sort.Slice(models, func(i, j int) bool {
return models[i].EstimatedCost > models[j].EstimatedCost
})
return models
}
// AggregateProjects computes per-project statistics from sessions.
func AggregateProjects(sessions []model.SessionStats, since, until time.Time) []model.ProjectStats {
filtered := FilterByTime(sessions, since, until)
projMap := make(map[string]*model.ProjectStats)
for _, s := range filtered {
ps, ok := projMap[s.Project]
if !ok {
ps = &model.ProjectStats{Project: s.Project}
projMap[s.Project] = ps
}
ps.Sessions++
ps.Prompts += s.UserMessages
ps.TotalTokens += s.InputTokens + s.OutputTokens +
s.CacheCreation5mTokens + s.CacheCreation1hTokens
ps.EstimatedCost += s.EstimatedCost
}
// Sort by cost descending
projects := make([]model.ProjectStats, 0, len(projMap))
for _, ps := range projMap {
projects = append(projects, *ps)
}
sort.Slice(projects, func(i, j int) bool {
return projects[i].EstimatedCost > projects[j].EstimatedCost
})
return projects
}
// AggregateHourly computes prompt counts by hour of day.
func AggregateHourly(sessions []model.SessionStats, since, until time.Time) []model.HourlyStats {
filtered := FilterByTime(sessions, since, until)
hours := make([]model.HourlyStats, 24)
for i := range hours {
hours[i].Hour = i
}
// We attribute all prompts and tokens to the session's start hour
for _, s := range filtered {
if s.StartTime.IsZero() {
continue
}
h := s.StartTime.Local().Hour()
hours[h].Prompts += s.UserMessages
hours[h].Sessions++
hours[h].Tokens += s.InputTokens + s.OutputTokens
}
return hours
}
// FilterByTime returns sessions whose start time falls within [since, until).
func FilterByTime(sessions []model.SessionStats, since, until time.Time) []model.SessionStats {
if since.IsZero() && until.IsZero() {
return sessions
}
var result []model.SessionStats
for _, s := range sessions {
if s.StartTime.IsZero() {
continue
}
if !since.IsZero() && s.StartTime.Before(since) {
continue
}
if !until.IsZero() && !s.StartTime.Before(until) {
continue
}
result = append(result, s)
}
return result
}
// FilterByProject returns sessions matching the project substring.
func FilterByProject(sessions []model.SessionStats, project string) []model.SessionStats {
if project == "" {
return sessions
}
var result []model.SessionStats
for _, s := range sessions {
if containsIgnoreCase(s.Project, project) {
result = append(result, s)
}
}
return result
}
// FilterByModel returns sessions that have at least one API call to the given model.
func FilterByModel(sessions []model.SessionStats, modelFilter string) []model.SessionStats {
if modelFilter == "" {
return sessions
}
var result []model.SessionStats
for _, s := range sessions {
for m := range s.Models {
if containsIgnoreCase(m, modelFilter) {
result = append(result, s)
break
}
}
}
return result
}
func containsIgnoreCase(s, substr string) bool {
return strings.Contains(strings.ToLower(s), strings.ToLower(substr))
}