Allow sending resources on key challenge, send multiple challenges to specific browsers

lib/challenge.go

@@ -96,7 +96,7 @@ func (state *State) IssueChallengeToken(name string, key, result []byte, until t
 	return token, nil
 }
 
-func (state *State) VerifyChallengeToken(name string, expectedKey []byte, r *http.Request) (ok bool, err error) {
+func (state *State) VerifyChallengeToken(name string, expectedKey []byte, w http.ResponseWriter, r *http.Request) (ok bool, err error) {
 	c, ok := state.Challenges[name]
 	if !ok {
 		return false, errors.New("challenge not found")
@@ -104,7 +104,22 @@ func (state *State) VerifyChallengeToken(name string, expectedKey []byte, r *htt
 
 	cookie, err := r.Cookie(CookiePrefix + name)
 	if err != nil {
-		return false, err
+		// fallback: fetch cookie from response
+		if setCookies, ok := w.Header()["Set-Cookie"]; ok {
+			for _, setCookie := range setCookies {
+				newCookie, err := http.ParseSetCookie(setCookie)
+				if err != nil {
+					continue
+				}
+				// keep processing to find last set cookie
+				if newCookie.Name == name {
+					cookie = newCookie
+				}
+			}
+		}
+		if cookie == nil {
+			return false, err
+		}
 	}
 
 	token, err := jwt.ParseSigned(cookie.Value, []jose.SignatureAlgorithm{jose.EdDSA})

lib/http.go

@@ -143,7 +143,7 @@ func (state *State) handleRequest(w http.ResponseWriter, r *http.Request) {
 
 					for _, challengeName := range rule.Challenges {
 						key := state.GetChallengeKeyForRequest(challengeName, expiry, r)
-						ok, err := state.VerifyChallengeToken(challengeName, key, r)
+						ok, err := state.VerifyChallengeToken(challengeName, key, w, r)
 						if !ok || err != nil {
 							if !errors.Is(err, http.ErrNoCookie) {
 								ClearCookie(CookiePrefix+challengeName, w)

lib/state.go

@@ -347,6 +347,21 @@ func NewState(p policy.Policy, settings StateSettings) (state *State, err error)
 		case "":
 		case "http":
 		case "key":
+			mimeType := p.Parameters["key-mime"]
+			if mimeType == "" {
+				mimeType = "text/html; charset=utf-8"
+			}
+
+			httpCode, _ := strconv.Atoi(p.Parameters["key-code"])
+			if httpCode == 0 {
+				httpCode = http.StatusTemporaryRedirect
+			}
+
+			var content []byte
+			if data, ok := p.Parameters["key-content"]; ok {
+				content = []byte(data)
+			}
+
 			c.Verify = func(key []byte, result string) (bool, error) {
 				resultBytes, err := hex.DecodeString(result)
 				if err != nil {
@@ -359,6 +374,49 @@ func NewState(p policy.Policy, settings StateSettings) (state *State, err error)
 				return true, nil
 			}
 
+			c.VerifyChallenge = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+
+				err := func() (err error) {
+					expiry := time.Now().UTC().Add(DefaultValidity).Round(DefaultValidity)
+
+					key := state.GetChallengeKeyForRequest(challengeName, expiry, r)
+					result := r.FormValue("result")
+
+					if ok, err := c.Verify(key, result); err != nil {
+						return err
+					} else if !ok {
+						ClearCookie(CookiePrefix+challengeName, w)
+						_ = state.errorPage(w, http.StatusForbidden, fmt.Errorf("access denied: failed challenge %s", challengeName))
+						return nil
+					}
+
+					token, err := state.IssueChallengeToken(challengeName, key, []byte(result), expiry)
+					if err != nil {
+						ClearCookie(CookiePrefix+challengeName, w)
+					} else {
+						SetCookie(CookiePrefix+challengeName, token, expiry, w)
+					}
+
+					switch httpCode {
+					case http.StatusMovedPermanently, http.StatusFound, http.StatusSeeOther, http.StatusTemporaryRedirect, http.StatusPermanentRedirect:
+						http.Redirect(w, r, r.FormValue("redirect"), httpCode)
+					default:
+						w.Header().Set("Content-Type", mimeType)
+						w.WriteHeader(httpCode)
+						if content != nil {
+							_, _ = w.Write(content)
+						}
+					}
+
+					return nil
+				}()
+				if err != nil {
+					ClearCookie(CookiePrefix+challengeName, w)
+					_ = state.errorPage(w, http.StatusInternalServerError, err)
+					return
+				}
+			})
+
 		case "wasm":
 			wasmData, err := go_away.ChallengeFs.ReadFile(fmt.Sprintf("challenge/%s/runtime/%s", challengeName, p.Runtime.Asset))
 			if err != nil {

policy.yml

@@ -127,6 +127,11 @@ challenges:
       # verifies that result = key
       mode: "key"
       probability: 0.1
+    parameters:
+      key-code: 200
+      key-mime: text/css
+      key-content: ""
+
 
   # Verifies the existence of a cookie and confirms it against some backend request, passing the entire client cookie contents
   http-cookie-check:
@@ -185,6 +190,33 @@ conditions:
     - 'userAgent.startsWith("Go-http-client/")'
     - 'userAgent.startsWith("node-fetch/")'
     - 'userAgent.startsWith("reqwest/")'
+  is-suspicious-crawler:
+    - 'userAgent.contains("Presto/") || userAgent.contains("Trident/")'
+    # Old IE browsers
+    - 'userAgent.matches("MSIE ([2-9]|10|11)\\.")'
+    # Old Linux browsers
+    - 'userAgent.contains("Linux i686")'
+    # Old Windows browsers
+    - 'userAgent.matches("Windows (3|95|98|CE)") || userAgent.matches("Windows NT [1-5]\\.")'
+    # Old mobile browsers
+    - 'userAgent.matches("Android [1-5]\\.") || userAgent.matches("(iPad|iPhone) OS [1-9]_")'
+    # Old generic browsers
+    - 'userAgent.startsWith("Opera/")'
+    #- 'userAgent.matches("Gecko/(201[0-9]|200[0-9])")'
+    - 'userAgent.matches("^Mozilla/[1-4]")'
+  is-heavy-resource:
+    - 'path.startsWith("/explore/")'
+    - 'path.matches("^/[^/]+/[^/]+/src/commit/")'
+    - 'path.matches("^/[^/]+/[^/]+/compare/")'
+    - 'path.matches("^/[^/]+/[^/]+/commits/commit/")'
+    - 'path.matches("^/[^/]+/[^/]+/blame/")'
+    - 'path.matches("^/[^/]+/[^/]+/search/")'
+    - 'path.matches("^/[^/]+/[^/]+/find/")'
+    - 'path.matches("^/[^/]+/[^/]+/activity")'
+    # any search with a custom query
+    - '"q" in query && query.q != ""'
+    # user activity tab
+    - 'path.matches("^/[^/]") && "tab" in query && query.tab == "activity"'
 
 rules:
   - name: undesired-networks
@@ -215,33 +247,19 @@ rules:
       - 'userAgent.contains("Amazonbot") || userAgent.contains("Google-Extended") || userAgent.contains("PanguBot") || userAgent.contains("AI2Bot") || userAgent.contains("Diffbot") || userAgent.contains("cohere-training-data-crawler") || userAgent.contains("Applebot-Extended")'
     action: deny
 
-  - name: suspicious-crawlers
-    conditions:
-      - 'userAgent.contains("Presto/") || userAgent.contains("Trident/")'
-      # Old IE browsers
-      - 'userAgent.matches("MSIE ([2-9]|10|11)\\.")'
-      # Old Linux browsers
-      - 'userAgent.contains("Linux i686")'
-      # Old Windows browsers
-      - 'userAgent.matches("Windows (3|95|98|CE)") || userAgent.matches("Windows NT [1-5]\\.")'
-      # Old mobile browsers
-      - 'userAgent.matches("Android [1-5]\\.") || userAgent.matches("(iPad|iPhone) OS [1-9]_")'
-      # Old generic browsers
-      - 'userAgent.startsWith("Opera/")'
-      #- 'userAgent.matches("Gecko/(201[0-9]|200[0-9])")'
-      - 'userAgent.matches("^Mozilla/[1-4]")'
-    # check to continue below
+  # check a sequence of challenges for non logged in
+  - name: suspicious-crawlers/0
+    conditions: ['($is-suspicious-crawler)']
     action: check
     challenges: [js-pow-sha256, http-cookie-check]
-
-  - name: always-pow-challenge
-    conditions:
-      - 'path.startsWith("/user/sign_up") || path.startsWith("/user/login")'
-      # Match archive downloads from browsers and not tools
-      - 'path.matches("^/[^/]+/[^/]+/archive/.*\\.(bundle|zip|tar\\.gz)") && ($is-generic-browser)'
-    action: challenge
-    challenges: [js-pow-sha256]
-
+  - name: suspicious-crawlers/1
+    conditions: ['($is-suspicious-crawler)']
+    action: check
+    challenges: [self-header-refresh]
+  - name: suspicious-crawlers/2
+    conditions: ['($is-suspicious-crawler)']
+    action: check
+    challenges: [self-resource-load]
 
   - name: allow-static-resources
     conditions:
@@ -311,23 +329,19 @@ rules:
       - 'userAgent.contains("facebookexternalhit/") || userAgent.contains("facebookcatalog/")'
 
 
-  - name: heavy-operations
+  # check a sequence of challenges
+  - name: heavy-operations/0
     action: check
-    # check we are logged in, or force PoW
     challenges: [js-pow-sha256, http-cookie-check]
-    conditions:
-      - 'path.startsWith("/explore/")'
-      - 'path.matches("^/[^/]+/[^/]+/src/commit/")'
-      - 'path.matches("^/[^/]+/[^/]+/compare/")'
-      - 'path.matches("^/[^/]+/[^/]+/commits/commit/")'
-      - 'path.matches("^/[^/]+/[^/]+/blame/")'
-      - 'path.matches("^/[^/]+/[^/]+/search/")'
-      - 'path.matches("^/[^/]+/[^/]+/find/")'
-      - 'path.matches("^/[^/]+/[^/]+/activity")'
-      # any search with a custom query
-      - '"q" in query && query.q != ""'
-      # user activity tab
-      - 'path.matches("^/[^/]") && "tab" in query && query.tab == "activity"'
+    conditions: ['($is-heavy-resource)']
+  - name: heavy-operations/1
+    action: check
+    challenges: [self-header-refresh, http-cookie-check]
+    conditions: ['($is-heavy-resource)']
+  - name: heavy-operations/2
+    action: check
+    challenges: [self-resource-load, http-cookie-check]
+    conditions: ['($is-heavy-resource)']
 
   # Allow all source downloads not caught in browser above
   # todo: limit this as needed?
@@ -347,7 +361,7 @@ rules:
 
   - name: standard-browser
     action: challenge
-    challenges: [http-cookie-check, self-meta-refresh, js-pow-sha256]
+    challenges: [http-cookie-check, self-resource-load, self-meta-refresh, js-pow-sha256]
     conditions:
       - '($is-generic-browser)'