110 lines
3.1 KiB
Go
110 lines
3.1 KiB
Go
// Package scraper fetches the root-me.org scoreboard (Rankings) pages through
|
|
// an Anubis-solving HTTP client and parses them into scoreboard entries.
|
|
package scraper
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"log/slog"
|
|
"net/http"
|
|
"sort"
|
|
|
|
"git.mkz.me/mycroft/root-me-api/internal/scoreboard"
|
|
)
|
|
|
|
// pageSize is the number of rows per rankings page (the debut_classement step).
|
|
const pageSize = 50
|
|
|
|
// rankingsURL is the public, login-free ajax fragment that renders the
|
|
// Rankings table. %d is the debut_classement offset.
|
|
const rankingsURL = "https://www.root-me.org/?page=structure&inc=modeles/classement&lang=en&ajah=1&debut_classement=%d"
|
|
|
|
// Doer is the subset of *http.Client the scraper needs; the anubis.Transport's
|
|
// client satisfies it.
|
|
type Doer interface {
|
|
Do(*http.Request) (*http.Response, error)
|
|
}
|
|
|
|
// Scraper fetches and parses scoreboard pages.
|
|
type Scraper struct {
|
|
Client Doer
|
|
Logger *slog.Logger
|
|
}
|
|
|
|
// New returns a Scraper using the given HTTP client.
|
|
func New(client Doer, logger *slog.Logger) *Scraper {
|
|
if logger == nil {
|
|
logger = slog.Default()
|
|
}
|
|
return &Scraper{Client: client, Logger: logger}
|
|
}
|
|
|
|
// Fetch retrieves the first n pages of the scoreboard and returns the merged,
|
|
// rank-sorted entries. Pages are fetched sequentially to share one Anubis solve
|
|
// and to stay gentle on root-me / its rate limiter.
|
|
func (s *Scraper) Fetch(ctx context.Context, pages int) ([]scoreboard.Entry, error) {
|
|
if pages <= 0 {
|
|
pages = 1
|
|
}
|
|
var all []scoreboard.Entry
|
|
for p := range pages {
|
|
offset := p * pageSize
|
|
entries, err := s.fetchPage(ctx, offset)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("scraper: page %d (offset %d): %w", p+1, offset, err)
|
|
}
|
|
s.Logger.Debug("fetched rankings page", "page", p+1, "offset", offset, "rows", len(entries))
|
|
all = append(all, entries...)
|
|
}
|
|
|
|
all = dedupeByUsername(all)
|
|
// root-me returns rows in score-descending order; sort defensively (stable,
|
|
// so ties keep scrape order) and assign canonical 1-based ranks.
|
|
sort.SliceStable(all, func(i, j int) bool { return all[i].Score > all[j].Score })
|
|
for i := range all {
|
|
all[i].Rank = i + 1
|
|
}
|
|
return all, nil
|
|
}
|
|
|
|
func (s *Scraper) fetchPage(ctx context.Context, offset int) ([]scoreboard.Entry, error) {
|
|
u := fmt.Sprintf(rankingsURL, offset)
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
req.Header.Set("Accept", "text/html, */*; q=0.01")
|
|
req.Header.Set("X-Requested-With", "XMLHttpRequest")
|
|
|
|
resp, err := s.Client.Do(req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
body, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read body: %w", err)
|
|
}
|
|
if resp.StatusCode != http.StatusOK {
|
|
return nil, fmt.Errorf("unexpected status %d", resp.StatusCode)
|
|
}
|
|
return parseRankings(string(body))
|
|
}
|
|
|
|
// dedupeByUsername drops duplicate users (defensive against overlapping pages),
|
|
// keeping the first occurrence.
|
|
func dedupeByUsername(in []scoreboard.Entry) []scoreboard.Entry {
|
|
seen := make(map[string]struct{}, len(in))
|
|
out := in[:0]
|
|
for _, e := range in {
|
|
if _, dup := seen[e.Username]; dup {
|
|
continue
|
|
}
|
|
seen[e.Username] = struct{}{}
|
|
out = append(out, e)
|
|
}
|
|
return out
|
|
}
|