125 lines
3.4 KiB
Go
125 lines
3.4 KiB
Go
package scraper
|
|
|
|
import (
|
|
"fmt"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"github.com/PuerkitoBio/goquery"
|
|
|
|
"git.mkz.me/mycroft/root-me-api/internal/scoreboard"
|
|
)
|
|
|
|
// The rankings ajax fragment is a single <table> whose rows each have 6 <td>:
|
|
//
|
|
// [0] "# <rank>"
|
|
// [1] avatar <img alt="<username>">
|
|
// [2] <a href="/<user>?lang=en" title="profil of <user>"><user></a>
|
|
// [3] country flag <img src=".../pays/<cc>.svg">
|
|
// [4] grade icon <img src=".../rang/<grade>.svg" alt="<grade>">
|
|
// [5] score <a href="<user>?inc=score" title="<big>"><visible-score></a>
|
|
//
|
|
// Selectors are intentionally lenient (match on structure/attributes rather
|
|
// than volatile CSS classes) so minor template changes don't break parsing.
|
|
|
|
var (
|
|
rankRe = regexp.MustCompile(`#?\s*(\d+)`)
|
|
flagRe = regexp.MustCompile(`/pays/([a-zA-Z]{2})\.svg`)
|
|
gradeRe = regexp.MustCompile(`/rang/([a-zA-Z0-9_-]+)\.svg`)
|
|
nonDigits = regexp.MustCompile(`\D+`)
|
|
)
|
|
|
|
// parseRankings parses one rankings fragment into entries.
|
|
func parseRankings(html string) ([]scoreboard.Entry, error) {
|
|
doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("scraper: parse html: %w", err)
|
|
}
|
|
|
|
var entries []scoreboard.Entry
|
|
var rowErr error
|
|
|
|
doc.Find("table tr").Each(func(_ int, tr *goquery.Selection) {
|
|
tds := tr.Find("td")
|
|
if tds.Length() < 6 {
|
|
return // header row (uses <td>Position</td>... but lacks the score link) or layout row
|
|
}
|
|
e, ok, err := parseRow(tds)
|
|
if err != nil {
|
|
rowErr = err
|
|
return
|
|
}
|
|
if ok {
|
|
entries = append(entries, e)
|
|
}
|
|
})
|
|
if rowErr != nil {
|
|
return nil, rowErr
|
|
}
|
|
if len(entries) == 0 {
|
|
return nil, fmt.Errorf("scraper: no rows parsed (fragment layout may have changed)")
|
|
}
|
|
return entries, nil
|
|
}
|
|
|
|
func parseRow(tds *goquery.Selection) (scoreboard.Entry, bool, error) {
|
|
var e scoreboard.Entry
|
|
|
|
// [0] rank — header row's first cell is "Position" (no digits) → skip.
|
|
rankText := strings.TrimSpace(tds.Eq(0).Text())
|
|
m := rankRe.FindStringSubmatch(rankText)
|
|
if m == nil {
|
|
return e, false, nil // not a data row
|
|
}
|
|
rank, err := strconv.Atoi(m[1])
|
|
if err != nil {
|
|
return e, false, fmt.Errorf("scraper: bad rank %q: %w", rankText, err)
|
|
}
|
|
e.Rank = rank
|
|
|
|
// [2] username + profile path.
|
|
link := tds.Eq(2).Find("a").First()
|
|
e.Username = strings.TrimSpace(link.Text())
|
|
if e.Username == "" {
|
|
// Fall back to avatar alt in [1].
|
|
e.Username = strings.TrimSpace(tds.Eq(1).Find("img").AttrOr("alt", ""))
|
|
}
|
|
if href, ok := link.Attr("href"); ok {
|
|
if p := strings.SplitN(href, "?", 2)[0]; p != "" {
|
|
e.ProfilePath = p
|
|
}
|
|
}
|
|
if e.Username == "" {
|
|
return e, false, fmt.Errorf("scraper: row rank %d has no username", rank)
|
|
}
|
|
|
|
// [3] country flag.
|
|
if src, ok := tds.Eq(3).Find("img").Attr("src"); ok {
|
|
if fm := flagRe.FindStringSubmatch(src); fm != nil {
|
|
e.Country = strings.ToLower(fm[1])
|
|
}
|
|
}
|
|
|
|
// [4] grade — prefer the icon's alt, else derive from the svg filename.
|
|
grade := tds.Eq(4).Find("img").AttrOr("alt", "")
|
|
if grade == "" {
|
|
if src, ok := tds.Eq(4).Find("img").Attr("src"); ok {
|
|
if gm := gradeRe.FindStringSubmatch(src); gm != nil {
|
|
grade = gm[1]
|
|
}
|
|
}
|
|
}
|
|
e.Grade = strings.TrimSpace(grade)
|
|
|
|
// [5] score — visible text of the link.
|
|
scoreText := nonDigits.ReplaceAllString(tds.Eq(5).Text(), "")
|
|
if scoreText != "" {
|
|
if e.Score, err = strconv.Atoi(scoreText); err != nil {
|
|
return e, false, fmt.Errorf("scraper: bad score %q for rank %d: %w", scoreText, rank, err)
|
|
}
|
|
}
|
|
|
|
return e, true, nil
|
|
}
|