feat: initial commit

internal/anubis/challenge.go

@@ -0,0 +1,111 @@
+package anubis
+
+import (
+	"encoding/json"
+	"fmt"
+	"regexp"
+)
+
+// Anubis embeds several JSON blobs in its interstitial as
+// <script id="...">{...}</script> elements. We read two:
+//
+//   - anubis_challenge: the canonical challenge (id, randomData, rules)
+//   - preact_info:      the preact frontend's view, crucially the `redir`
+//     (the fully-formed pass-challenge URL) and `challenge` string.
+//
+// root-me.org uses the "preact" challenge: the answer is simply
+// SHA-256(randomData) (no proof-of-work nonce), gated by a minimum wait of
+// difficulty*80ms server-side.
+type anubisChallenge struct {
+	Challenge struct {
+		ID         string `json:"id"`
+		Method     string `json:"method"`
+		RandomData string `json:"randomData"`
+		IssuedAt   string `json:"issuedAt"`
+	} `json:"challenge"`
+	Rules struct {
+		Difficulty int    `json:"difficulty"`
+		Algorithm  string `json:"algorithm"`
+	} `json:"rules"`
+}
+
+type preactInfo struct {
+	Challenge  string `json:"challenge"`
+	Difficulty int    `json:"difficulty"`
+	Redir      string `json:"redir"` // relative pass-challenge URL with id+redir baked in
+}
+
+// challenge is the normalized view the transport acts on.
+type challenge struct {
+	algorithm  string
+	id         string
+	randomData string
+	difficulty int
+	redir      string // preact: server-provided relative pass-challenge URL
+}
+
+func scriptRe(id string) *regexp.Regexp {
+	return regexp.MustCompile(
+		`(?is)<script[^>]*\bid=["']` + regexp.QuoteMeta(id) + `["'][^>]*>(.*?)</script>`)
+}
+
+var (
+	anubisChallengeRe = scriptRe("anubis_challenge")
+	preactInfoRe      = scriptRe("preact_info")
+)
+
+// isInterstitial reports whether an HTML body is an Anubis challenge page.
+func isInterstitial(body []byte) bool {
+	return anubisChallengeRe.Match(body)
+}
+
+func extractJSON(re *regexp.Regexp, body []byte, into any) error {
+	m := re.FindSubmatch(body)
+	if m == nil {
+		return fmt.Errorf("anubis: script element not found")
+	}
+	if err := json.Unmarshal(m[1], into); err != nil {
+		return fmt.Errorf("anubis: decode embedded JSON: %w", err)
+	}
+	return nil
+}
+
+// parseChallenge extracts and normalizes the Anubis challenge from
+// interstitial HTML.
+func parseChallenge(body []byte) (*challenge, error) {
+	var ac anubisChallenge
+	if err := extractJSON(anubisChallengeRe, body, &ac); err != nil {
+		return nil, fmt.Errorf("anubis: parse anubis_challenge: %w", err)
+	}
+	if ac.Challenge.RandomData == "" || ac.Rules.Difficulty <= 0 {
+		return nil, fmt.Errorf("anubis: incomplete challenge (randomData=%q difficulty=%d)",
+			ac.Challenge.RandomData, ac.Rules.Difficulty)
+	}
+
+	c := &challenge{
+		algorithm:  ac.Rules.Algorithm,
+		id:         ac.Challenge.ID,
+		randomData: ac.Challenge.RandomData,
+		difficulty: ac.Rules.Difficulty,
+	}
+
+	// The preact challenge carries the fully-formed pass-challenge URL in
+	// preact_info.redir; grab it when present.
+	if c.algorithm == "preact" {
+		var pi preactInfo
+		if err := extractJSON(preactInfoRe, body, &pi); err != nil {
+			return nil, fmt.Errorf("anubis: parse preact_info: %w", err)
+		}
+		if pi.Redir == "" {
+			return nil, fmt.Errorf("anubis: preact_info missing redir")
+		}
+		c.redir = pi.Redir
+		if pi.Challenge != "" {
+			c.randomData = pi.Challenge
+		}
+		if pi.Difficulty > 0 {
+			c.difficulty = pi.Difficulty
+		}
+	}
+	return c, nil
+}

internal/anubis/http_helpers.go

@@ -0,0 +1,52 @@
+package anubis
+
+import (
+	"bytes"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+)
+
+// resolveRef resolves a (possibly relative) URL reference against a base URL,
+// preserving the base's scheme and host. Anubis's preact redir is host-relative
+// (e.g. "/.within.website/...").
+func resolveRef(base *url.URL, ref string) *url.URL {
+	u, err := url.Parse(ref)
+	if err != nil {
+		// Fall back to a copy of base; the caller's cookie check will catch it.
+		c := *base
+		return &c
+	}
+	return base.ResolveReference(u)
+}
+
+// maybeHTML reports whether a response could be an HTML page (and thus possibly
+// an Anubis interstitial). Non-HTML responses are passed through untouched.
+func maybeHTML(resp *http.Response) bool {
+	ct := resp.Header.Get("Content-Type")
+	if ct == "" {
+		return true // unknown; be safe and inspect
+	}
+	return strings.Contains(strings.ToLower(ct), "text/html")
+}
+
+// withBody returns resp with its body replaced by an in-memory reader over b,
+// so the (already consumed) body can be read again by the caller.
+func withBody(resp *http.Response, b []byte) *http.Response {
+	resp.Body = io.NopCloser(bytes.NewReader(b))
+	resp.ContentLength = int64(len(b))
+	resp.Header.Del("Content-Length")
+	return resp
+}
+
+// applyCookies replaces the request's Cookie header with the cookies the shared
+// jar holds for the request URL. The replayed request goes through the base
+// transport directly (bypassing the client jar), so we must attach the freshly
+// obtained auth cookie ourselves.
+func (t *Transport) applyCookies(req *http.Request) {
+	req.Header.Del("Cookie")
+	for _, c := range t.Jar.Cookies(req.URL) {
+		req.AddCookie(c)
+	}
+}

internal/anubis/pow.go

@@ -0,0 +1,63 @@
+// Package anubis implements a native Go client for getting past the Anubis
+// (TecharoHQ) proof-of-work anti-scraper proxy without a headless browser.
+//
+// The challenge is embedded in the interstitial HTML as a JSON document inside
+// a <script id="anubis_challenge"> element. The client must find an integer
+// nonce such that SHA-256(randomData + nonce) has `difficulty` leading hex-zero
+// digits, then submit it to the pass-challenge endpoint to obtain a signed-JWT
+// auth cookie.
+package anubis
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"strconv"
+)
+
+// sha256Hex returns the hex-encoded SHA-256 of s. This is the answer to a
+// "preact" Anubis challenge: result = SHA256(randomData).
+func sha256Hex(s string) string {
+	sum := sha256.Sum256([]byte(s))
+	return hex.EncodeToString(sum[:])
+}
+
+// Solve performs the Anubis proof-of-work: it searches for the smallest nonce
+// (starting at 0) such that SHA-256(randomData + nonce) has `difficulty`
+// leading hex-zero digits. It returns the hex-encoded hash and the nonce.
+//
+// The zero check mirrors the Anubis worker: the first floor(difficulty/2)
+// bytes must be zero and, when difficulty is odd, the high nibble of the next
+// byte must also be zero.
+func Solve(randomData string, difficulty int) (hash string, nonce uint64) {
+	prefix := []byte(randomData)
+	zeroBytes := difficulty / 2
+	oddNibble := difficulty%2 != 0
+
+	// Reusable buffer: challenge prefix + decimal nonce, refilled each round.
+	buf := make([]byte, 0, len(prefix)+20)
+
+	for n := uint64(0); ; n++ {
+		buf = append(buf[:0], prefix...)
+		buf = strconv.AppendUint(buf, n, 10)
+		sum := sha256.Sum256(buf)
+
+		if leadingZeros(sum[:], zeroBytes, oddNibble) {
+			return hex.EncodeToString(sum[:]), n
+		}
+	}
+}
+
+// leadingZeros reports whether sum has the required number of leading hex-zero
+// digits: zeroBytes fully-zero bytes, plus (when oddNibble) a zero high nibble
+// in the following byte.
+func leadingZeros(sum []byte, zeroBytes int, oddNibble bool) bool {
+	for i := range zeroBytes {
+		if sum[i] != 0 {
+			return false
+		}
+	}
+	if oddNibble {
+		return sum[zeroBytes]>>4 == 0
+	}
+	return true
+}

internal/anubis/pow_test.go

@@ -0,0 +1,58 @@
+package anubis
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"strconv"
+	"strings"
+	"testing"
+)
+
+func TestSolveProducesLeadingZeros(t *testing.T) {
+	cases := []struct {
+		randomData string
+		difficulty int
+	}{
+		{"abc123", 1},
+		{"deadbeefcafebabe", 2},
+		{"root-me-scoreboard", 3},
+		{"odd-difficulty-check", 5},
+	}
+
+	for _, tc := range cases {
+		hash, nonce := Solve(tc.randomData, tc.difficulty)
+
+		// The hash must have exactly `difficulty` leading '0' hex digits.
+		if got := leadingZeroDigits(hash); got < tc.difficulty {
+			t.Errorf("Solve(%q, %d): hash %s has %d leading zeros, want >= %d",
+				tc.randomData, tc.difficulty, hash, got, tc.difficulty)
+		}
+
+		// And it must be reproducible: SHA-256(randomData + nonce) == hash.
+		want := sha256.Sum256([]byte(tc.randomData + strconv.FormatUint(nonce, 10)))
+		if hex.EncodeToString(want[:]) != hash {
+			t.Errorf("Solve(%q, %d): hash not reproducible from nonce %d",
+				tc.randomData, tc.difficulty, nonce)
+		}
+	}
+}
+
+func TestSolveFindsSmallestNonce(t *testing.T) {
+	// Brute the expected smallest nonce independently and compare.
+	const data, diff = "smallest", 2 // difficulty 2 == first byte zero
+	var want uint64
+	for {
+		sum := sha256.Sum256([]byte(data + strconv.FormatUint(want, 10)))
+		if sum[0] == 0 {
+			break
+		}
+		want++
+	}
+	if _, nonce := Solve(data, diff); nonce != want {
+		t.Errorf("Solve smallest nonce = %d, want %d", nonce, want)
+	}
+}
+
+func leadingZeroDigits(hexHash string) int {
+	return len(hexHash) - len(strings.TrimLeft(hexHash, "0"))
+}

internal/anubis/transport.go

@@ -0,0 +1,269 @@
+package anubis
+
+import (
+	"fmt"
+	"io"
+	"log/slog"
+	"net/http"
+	"net/http/cookiejar"
+	"net/url"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"golang.org/x/sync/singleflight"
+)
+
+const (
+	// AuthCookie is the signed-JWT cookie Anubis sets once a challenge passes.
+	AuthCookie = "techaro.lol-anubis-auth"
+	// passChallengePath is the endpoint that validates a solved challenge.
+	passChallengePath = "/.within.website/x/cmd/anubis/api/pass-challenge"
+	// DefaultUserAgent is a stable, browser-like UA. The Anubis JWT is bound to
+	// request metadata (incl. User-Agent), so every request through this
+	// transport must use the same value.
+	DefaultUserAgent = "Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0"
+)
+
+// maxBodyPeek caps how much of a response body we read to detect/parse a
+// challenge. Anubis interstitials are small; real pages can be large, so we
+// only buffer when the response looks like an interstitial.
+const maxChallengePeek = 1 << 20 // 1 MiB
+
+// Transport is an http.RoundTripper that transparently solves Anubis
+// proof-of-work challenges. When a request hits an interstitial, it solves the
+// PoW, calls pass-challenge to obtain the auth cookie (stored in a shared
+// jar), then replays the original request.
+//
+// A singleflight group collapses concurrent solves so only one goroutine pays
+// the PoW cost while others wait for the resulting cookie.
+type Transport struct {
+	Base      http.RoundTripper
+	UserAgent string
+	Jar       http.CookieJar
+	Logger    *slog.Logger
+
+	sf singleflight.Group
+	mu sync.Mutex // guards client construction
+	hc *http.Client
+}
+
+// New returns a Transport with a fresh cookie jar and default settings.
+func New() (*Transport, error) {
+	jar, err := cookiejar.New(nil)
+	if err != nil {
+		return nil, fmt.Errorf("anubis: create cookie jar: %w", err)
+	}
+	return &Transport{
+		Base:      http.DefaultTransport,
+		UserAgent: DefaultUserAgent,
+		Jar:       jar,
+		Logger:    slog.Default(),
+	}, nil
+}
+
+// Client returns an *http.Client that uses this Transport and shares its cookie
+// jar, so callers benefit from both the auto-solving and the persisted cookie.
+func (t *Transport) Client() *http.Client {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	if t.hc == nil {
+		t.hc = &http.Client{Transport: t, Jar: t.Jar, Timeout: 30 * time.Second}
+	}
+	return t.hc
+}
+
+// RoundTrip implements http.RoundTripper.
+func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) {
+	t.setUA(req)
+
+	resp, err := t.base().RoundTrip(req)
+	if err != nil {
+		return nil, err
+	}
+
+	// Anubis serves interstitials as 200/text-html. Cheap content-type gate
+	// before we buffer anything.
+	if !maybeHTML(resp) {
+		return resp, nil
+	}
+
+	body, err := io.ReadAll(io.LimitReader(resp.Body, maxChallengePeek))
+	if err != nil {
+		resp.Body.Close()
+		return nil, fmt.Errorf("anubis: read body: %w", err)
+	}
+	resp.Body.Close()
+
+	if !isInterstitial(body) {
+		// Not a challenge; hand back a response with a replayable body.
+		return withBody(resp, body), nil
+	}
+
+	if t.Logger != nil {
+		t.Logger.Debug("anubis interstitial detected", "url", req.URL.String())
+	}
+	// The interstitial sets a cookie-verification cookie that pass-challenge
+	// requires. The outer http.Client only commits response cookies to the jar
+	// after RoundTrip returns, so capture them now — before we solve.
+	t.Jar.SetCookies(req.URL, resp.Cookies())
+
+	if err := t.solveFor(req, body); err != nil {
+		return nil, err
+	}
+
+	// Replay the original request now that we hold the auth cookie. The replay
+	// goes through the base transport directly, so attach jar cookies manually.
+	replay := req.Clone(req.Context())
+	t.setUA(replay)
+	t.applyCookies(replay)
+	resp2, err := t.base().RoundTrip(replay)
+	if err != nil {
+		return nil, err
+	}
+	if maybeHTML(resp2) {
+		body2, err := io.ReadAll(io.LimitReader(resp2.Body, maxChallengePeek))
+		resp2.Body.Close()
+		if err != nil {
+			return nil, fmt.Errorf("anubis: read replayed body: %w", err)
+		}
+		if isInterstitial(body2) {
+			return nil, fmt.Errorf("anubis: still challenged after passing challenge for %s", req.URL)
+		}
+		return withBody(resp2, body2), nil
+	}
+	return resp2, nil
+}
+
+// solveFor parses the challenge from body, computes the answer, and calls
+// pass-challenge to install the auth cookie. Concurrent calls for the same
+// host are collapsed via singleflight.
+func (t *Transport) solveFor(req *http.Request, body []byte) error {
+	key := req.URL.Host
+	_, err, _ := t.sf.Do(key, func() (any, error) {
+		c, err := parseChallenge(body)
+		if err != nil {
+			return nil, err
+		}
+		passURL, err := t.answer(req, c)
+		if err != nil {
+			return nil, err
+		}
+		return nil, t.passChallenge(req, passURL)
+	})
+	return err
+}
+
+// answer computes the pass-challenge URL for the given challenge, performing
+// any required wait or proof-of-work. It supports the "preact" challenge (a
+// plain SHA-256 of randomData plus a timing gate) and the "fast"/"slow"
+// proof-of-work challenges (nonce search).
+func (t *Transport) answer(orig *http.Request, c *challenge) (string, error) {
+	start := time.Now()
+	switch c.algorithm {
+	case "preact":
+		// Answer is SHA-256(randomData); the server enforces a minimum wait of
+		// difficulty*80ms. Mirror the frontend's difficulty*125ms delay.
+		result := sha256Hex(c.randomData)
+		wait := time.Duration(c.difficulty) * 125 * time.Millisecond
+		time.Sleep(wait)
+
+		u := resolveRef(orig.URL, c.redir)
+		q := u.Query()
+		q.Set("result", result)
+		u.RawQuery = q.Encode()
+		if t.Logger != nil {
+			t.Logger.Info("answered anubis preact challenge",
+				"difficulty", c.difficulty, "waited", wait)
+		}
+		return u.String(), nil
+
+	case "fast", "slow", "proofofwork", "":
+		hash, nonce := Solve(c.randomData, c.difficulty)
+		u := &url.URL{Scheme: orig.URL.Scheme, Host: orig.URL.Host, Path: passChallengePath}
+		q := url.Values{
+			"id":          {c.id},
+			"response":    {hash},
+			"nonce":       {strconv.FormatUint(nonce, 10)},
+			"redir":       {orig.URL.RequestURI()},
+			"elapsedTime": {strconv.FormatInt(time.Since(start).Milliseconds(), 10)},
+		}
+		u.RawQuery = q.Encode()
+		if t.Logger != nil {
+			t.Logger.Info("solved anubis pow challenge",
+				"difficulty", c.difficulty, "nonce", nonce, "elapsed", time.Since(start))
+		}
+		return u.String(), nil
+
+	default:
+		return "", fmt.Errorf("anubis: unsupported challenge algorithm %q", c.algorithm)
+	}
+}
+
+// passChallenge GETs the pass-challenge URL so Anubis sets the auth cookie in
+// the shared jar.
+func (t *Transport) passChallenge(orig *http.Request, passURL string) error {
+	preq, err := http.NewRequestWithContext(orig.Context(), http.MethodGet, passURL, nil)
+	if err != nil {
+		return fmt.Errorf("anubis: build pass-challenge request: %w", err)
+	}
+	t.setUA(preq)
+	preq.Header.Set("Referer", orig.URL.String())
+
+	// Don't follow the post-pass redirect; we only need the Set-Cookie. Use a
+	// client bound to the shared jar so the cookie is captured.
+	hc := &http.Client{
+		Transport: t.base(),
+		Jar:       t.Jar,
+		CheckRedirect: func(*http.Request, []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+		Timeout: 30 * time.Second,
+	}
+	resp, err := hc.Do(preq)
+	if err != nil {
+		return fmt.Errorf("anubis: pass-challenge: %w", err)
+	}
+	defer resp.Body.Close()
+	io.Copy(io.Discard, io.LimitReader(resp.Body, maxChallengePeek))
+
+	if !t.hasAuthCookie(orig.URL) {
+		return fmt.Errorf("anubis: pass-challenge did not yield %s cookie (status %d)", AuthCookie, resp.StatusCode)
+	}
+	return nil
+}
+
+// hasAuthCookie reports whether the jar holds an Anubis auth cookie for u.
+// Deployments customize the cookie prefix (default "techaro.lol-anubis-auth",
+// root-me uses "anubis-cookie-auth"), so we match any non-empty cookie whose
+// name ends in "-auth" while excluding the "-cookie-verification" helper.
+func (t *Transport) hasAuthCookie(u *url.URL) bool {
+	for _, c := range t.Jar.Cookies(u) {
+		if c.Value == "" || strings.Contains(c.Name, "verification") {
+			continue
+		}
+		if c.Name == AuthCookie || strings.HasSuffix(c.Name, "-auth") {
+			return true
+		}
+	}
+	return false
+}
+
+func (t *Transport) base() http.RoundTripper {
+	if t.Base != nil {
+		return t.Base
+	}
+	return http.DefaultTransport
+}
+
+func (t *Transport) setUA(req *http.Request) {
+	ua := t.UserAgent
+	if ua == "" {
+		ua = DefaultUserAgent
+	}
+	req.Header.Set("User-Agent", ua)
+	if req.Header.Get("Accept-Language") == "" {
+		req.Header.Set("Accept-Language", "en-US,en;q=0.9")
+	}
+}