// Package scraper fetches the root-me.org scoreboard (Rankings) pages through // an Anubis-solving HTTP client and parses them into scoreboard entries. package scraper import ( "context" "fmt" "io" "log/slog" "net/http" "sort" "git.mkz.me/mycroft/root-me-api/internal/scoreboard" ) // pageSize is the number of rows per rankings page (the debut_classement step). const pageSize = 50 // rankingsURL is the public, login-free ajax fragment that renders the // Rankings table. %d is the debut_classement offset. const rankingsURL = "https://www.root-me.org/?page=structure&inc=modeles/classement&lang=en&ajah=1&debut_classement=%d" // Doer is the subset of *http.Client the scraper needs; the anubis.Transport's // client satisfies it. type Doer interface { Do(*http.Request) (*http.Response, error) } // Scraper fetches and parses scoreboard pages. type Scraper struct { Client Doer Logger *slog.Logger } // New returns a Scraper using the given HTTP client. func New(client Doer, logger *slog.Logger) *Scraper { if logger == nil { logger = slog.Default() } return &Scraper{Client: client, Logger: logger} } // Fetch retrieves the first n pages of the scoreboard and returns the merged, // rank-sorted entries. Pages are fetched sequentially to share one Anubis solve // and to stay gentle on root-me / its rate limiter. func (s *Scraper) Fetch(ctx context.Context, pages int) ([]scoreboard.Entry, error) { if pages <= 0 { pages = 1 } var all []scoreboard.Entry for p := range pages { offset := p * pageSize entries, err := s.fetchPage(ctx, offset) if err != nil { return nil, fmt.Errorf("scraper: page %d (offset %d): %w", p+1, offset, err) } s.Logger.Debug("fetched rankings page", "page", p+1, "offset", offset, "rows", len(entries)) all = append(all, entries...) } all = dedupeByUsername(all) // root-me returns rows in score-descending order; sort defensively (stable, // so ties keep scrape order) and assign canonical 1-based ranks. sort.SliceStable(all, func(i, j int) bool { return all[i].Score > all[j].Score }) for i := range all { all[i].Rank = i + 1 } return all, nil } func (s *Scraper) fetchPage(ctx context.Context, offset int) ([]scoreboard.Entry, error) { u := fmt.Sprintf(rankingsURL, offset) req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) if err != nil { return nil, err } req.Header.Set("Accept", "text/html, */*; q=0.01") req.Header.Set("X-Requested-With", "XMLHttpRequest") resp, err := s.Client.Do(req) if err != nil { return nil, err } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { return nil, fmt.Errorf("read body: %w", err) } if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("unexpected status %d", resp.StatusCode) } return parseRankings(string(body)) } // dedupeByUsername drops duplicate users (defensive against overlapping pages), // keeping the first occurrence. func dedupeByUsername(in []scoreboard.Entry) []scoreboard.Entry { seen := make(map[string]struct{}, len(in)) out := in[:0] for _, e := range in { if _, dup := seen[e.Username]; dup { continue } seen[e.Username] = struct{}{} out = append(out, e) } return out }