package terms

import (
	"bufio"
	"io"
	"log"
	"strings"
)

func Frequency(r io.Reader) (map[string]int, error) {
	out := make(map[string]int)

	// tokenize
	s := bufio.NewScanner(r)
	s.Split(bufio.ScanWords)

	for s.Scan() {
		// casefold
		lower := strings.ToLower(s.Text())
		log.Printf("processing %s\n", lower)

		// stem
		if stopWords[lower] {
			log.Println("skipping due to stopWord")
			continue
		}

		out[lower]++
	}

	if err := s.Err(); err != nil {
		return out, err
	}

	return out, nil
}
