package terms

import (
	"bufio"
	"io"
	"log/slog"
	"strings"
)

func Frequency(logger *slog.Logger, r io.Reader) (map[string]int, error) {
	logger = logger.With("sys", "terms")
	out := make(map[string]int)

	// tokenize
	s := bufio.NewScanner(r)
	s.Split(bufio.ScanWords)

	for s.Scan() {
		// casefold
		lower := strings.ToLower(s.Text())
		logger.Debug("processing", "term", lower)

		// stem
		if stopWords[lower] {
			logger.Debug("skipping", "term", lower)
			continue
		}

		out[lower]++
	}

	if err := s.Err(); err != nil {
		return out, err
	}

	return out, nil
}
