add js-audit preset, streamline trufflehog HTTP ingestion

bbot/modules/trufflehog.py

@@ -37,6 +37,7 @@ class Config(BaseModuleConfig):
     ]
 
     scope_distance_modifier = 2
+    _module_threads = 2
 
     async def setup_deps(self):
         self.config_file = self.config.get("config", "")
@@ -86,6 +87,8 @@ async def handle_event(self, event):
         if isinstance(event.data, dict):
             description = event.data.get("description", "")
 
+        path = None
+        stdin_data = None
         if event.type == "CODE_REPOSITORY":
             path = event.url
             module = "github-experimental"
@@ -100,12 +103,8 @@ async def handle_event(self, event):
             else:
                 module = "filesystem"
         elif event.type in ("HTTP_RESPONSE", "RAW_TEXT"):
-            module = "filesystem"
-            file_data = event.raw_response if event.type == "HTTP_RESPONSE" else event.data
-            # write the response to a tempfile
-            # this is necessary because trufflehog doesn't yet support reading from stdin
-            # https://github.com/trufflesecurity/trufflehog/issues/162
-            path = self.helpers.tempfile(file_data, pipe=False)
+            module = "stdin"
+            stdin_data = event.raw_response if event.type == "HTTP_RESPONSE" else event.data
 
         if event.type == "CODE_REPOSITORY":
             host = event.host
@@ -118,7 +117,7 @@ async def handle_event(self, event):
             rawv2_result,
             verified,
             source_metadata,
-        ) in self.execute_trufflehog(module, path):
+        ) in self.execute_trufflehog(module, path=path, stdin_data=stdin_data):
             verified_str = "Verified" if verified else "Possible"
             confidence = "CONFIRMED" if verified else "MEDIUM"
             data = {
@@ -127,6 +126,10 @@ async def handle_event(self, event):
             }
             if host:
                 data["host"] = host
+            if event.type == "HTTP_RESPONSE":
+                url = event.data.get("url", "")
+                if url:
+                    data["url"] = url
 
             data["severity"] = "HIGH"
             data["confidence"] = confidence
@@ -142,11 +145,7 @@ async def handle_event(self, event):
                 context=f'{{module}} searched {event.type} using "{module}" method and found {verified_str.lower()} secret ({{event.type}}): {raw_result}',
             )
 
-        # clean up the tempfile when we're done with it
-        if event.type in ("HTTP_RESPONSE", "RAW_TEXT"):
-            path.unlink(missing_ok=True)
-
-    async def execute_trufflehog(self, module, path=None, string=None):
+    async def execute_trufflehog(self, module, path=None, stdin_data=None):
         command = [
             "trufflehog",
             "--json",
@@ -169,17 +168,23 @@ async def execute_trufflehog(self, module, path=None, string=None):
         elif module == "filesystem":
             command.append("filesystem")
             command.append(path)
+        elif module == "stdin":
+            command.append("stdin")
         elif module == "github-experimental":
             command.append("github-experimental")
             command.append("--repo=" + path)
             command.append("--object-discovery")
             command.append("--delete-cached-data")
             command.append("--token=" + self.github_token)
 
-        stats_file = self.helpers.tempfile_tail(callback=partial(self.log_trufflehog_status, path))
+        run_kwargs = {}
+        if stdin_data is not None:
+            run_kwargs["input"] = stdin_data
+
+        stats_file = self.helpers.tempfile_tail(callback=partial(self.log_trufflehog_status, path or module))
         try:
             with open(stats_file, "w") as stats_fh:
-                async for line in self.helpers.run_live(command, stderr=stats_fh):
+                async for line in self.run_process_live(command, stderr=stats_fh, **run_kwargs):
                     try:
                         j = json.loads(line)
                     except json.decoder.JSONDecodeError:
@@ -202,7 +207,7 @@ async def execute_trufflehog(self, module, path=None, string=None):
         finally:
             stats_file.unlink(missing_ok=True)
 
-    def log_trufflehog_status(self, path, line):
+    def log_trufflehog_status(self, target, line):
         try:
             line = json.loads(line)
         except Exception:
@@ -211,5 +216,5 @@ def log_trufflehog_status(self, path, line):
         message = line.get("msg", "")
         ts = line.get("ts", "")
         status = f"Message: {message} | Timestamp: {ts}"
-        self.verbose(f"Current scan target: {path}")
+        self.verbose(f"Current scan target: {target}")
         self.verbose(status)

bbot/presets/web/js-audit.yml

@@ -0,0 +1,20 @@
+description: Hunt for leaked credentials and vulnerable libraries in client-side JavaScript
+
+
+modules:
+  - http
+  - trufflehog
+  - badsecrets
+  - retirejs
+  - robots
+  - wayback
+
+config:
+  modules:
+    trufflehog:
+      only_verified: false
+    robots:
+      include_sitemap: true
+    wayback:
+      urls: true
+      archive: true

bbot/test/test_step_2/module_tests/test_module_trufflehog.py

@@ -1328,7 +1328,11 @@ async def setup_before_prep(self, module_test):
         module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)
 
     def check(self, module_test, events):
-        assert any(e.type == "FINDING" for e in events)
+        findings = [e for e in events if e.type == "FINDING"]
+        assert findings, "trufflehog produced no FINDING for HTTP_RESPONSE with a secret"
+        assert all(f.data.get("url", "").startswith("http://127.0.0.1:8888") for f in findings), (
+            "FINDING must carry the source URL of the HTTP_RESPONSE, not a tempfile path"
+        )
 
 
 class TestTrufflehog_RAWText(ModuleTestBase):
@@ -1357,3 +1361,35 @@ def check(self, module_test, events):
         # Trufflehog emits HIGH severity and MEDIUM confidence for possible secrets
         assert finding_events[0].data["severity"] == "HIGH"
         assert finding_events[0].data["confidence"] == "MEDIUM"
+
+
+class TestTrufflehog_JSSecretURL(ModuleTestBase):
+    # A secret in a linked JS bundle must produce a FINDING whose url is the JS URL,
+    # not the HTML URL and not a tempfile path.
+    targets = ["http://127.0.0.1:8888"]
+    modules_overrides = ["http", "excavate", "trufflehog"]
+    config_overrides = {
+        "modules": {"trufflehog": {"only_verified": False}},
+        "web": {"spider_distance": 1, "spider_depth": 1},
+    }
+
+    # Split the webhook literal so the source doesn't match GitHub's push-protection
+    # secret-scanning regex; assembled at runtime, trufflehog still detects it.
+    _slack_webhook = "https://hooks.slack.com/services/T7KJ4NLXR/B8QZ" + "2MP3V/xJ9vNqLpZ4dKcYm2XwRfBg7T"
+
+    async def setup_before_prep(self, module_test):
+        module_test.set_expect_requests(
+            expect_args={"method": "GET", "uri": "/"},
+            respond_args={"response_data": '<html><script src="/app.js"></script></html>'},
+        )
+        module_test.set_expect_requests(
+            expect_args={"method": "GET", "uri": "/app.js"},
+            respond_args={"response_data": f'const w = "{self._slack_webhook}";'},
+        )
+
+    def check(self, module_test, events):
+        js_findings = [e for e in events if e.type == "FINDING" and "SlackWebhook" in e.data.get("description", "")]
+        assert js_findings, "trufflehog produced no SlackWebhook FINDING from linked JS"
+        assert all(f.data.get("url", "") == "http://127.0.0.1:8888/app.js" for f in js_findings), (
+            f"FINDING must carry the JS URL, got: {[f.data.get('url') for f in js_findings]}"
+        )

docs/scanning/presets_list.md

@@ -199,6 +199,38 @@ Category: web
 
 Modules: [0]("")
 
+## **js-audit**
+
+Hunt for leaked credentials and vulnerable libraries in client-side JavaScript
+
+??? note "`js-audit.yml`"
+    ```yaml title="~/.bbot/presets/web/js-audit.yml"
+    description: Hunt for leaked credentials and vulnerable libraries in client-side JavaScript
+
+
+    modules:
+      - http
+      - trufflehog
+      - badsecrets
+      - retirejs
+      - robots
+      - wayback
+
+    config:
+      modules:
+        trufflehog:
+          only_verified: false
+        robots:
+          include_sitemap: true
+        wayback:
+          urls: true
+          archive: true
+    ```
+
+Category: web
+
+Modules: [0]("")
+
 ## **kitchen-sink**
 
 Everything everywhere all at once
@@ -991,6 +1023,7 @@ Here is a the same data, but in a table:
 | email-enum        |            | Enumerate email addresses from APIs, web crawling, etc.                                                                                                                                                                                                                                                                                                                                                                                                                                                 | 0           |                                                                                                    |
 | fast              |            | Scan only the provided targets as fast as possible - no extra discovery                                                                                                                                                                                                                                                                                                                                                                                                                                 | 0           |                                                                                                    |
 | iis-shortnames    | web        | Recursively enumerate IIS shortnames                                                                                                                                                                                                                                                                                                                                                                                                                                                                    | 0           |                                                                                                    |
+| js-audit          | web        | Hunt for leaked credentials and vulnerable libraries in client-side JavaScript                                                                                                                                                                                                                                                                                                                                                                                                                          | 6           | badsecrets, http, retirejs, robots, trufflehog, wayback                                            |
 | kitchen-sink      |            | Everything everywhere all at once                                                                                                                                                                                                                                                                                                                                                                                                                                                                       | 7           | baddns, baddns_direct, baddns_zone, http, hunt, reflected_parameters, webbrute                     |
 | lightfuzz         | web        | Default fuzzing: all 9 submodules (cmdi, crypto, path, serial, sqli, ssti, xss, esi, ssrf) plus companion modules (badsecrets, hunt, reflected_parameters). POST fuzzing disabled but try_post_as_get enabled, so POST params are retested as GET. Skips confirmed WAFs.                                                                                                                                                                                                                                | 6           | badsecrets, http, hunt, lightfuzz, portfilter, reflected_parameters                                |
 | lightfuzz-heavy   | web        | Aggressive fuzzing: everything in lightfuzz, plus paramminer brute-force parameter discovery (headers, GET params, cookies), POST request fuzzing enabled, try_get_as_post enabled (GET params retested as POST), and robots.txt parsing. Still skips confirmed WAFs.                                                                                                                                                                                                                                   | 8           | badsecrets, http, hunt, lightfuzz, portfilter, reflected_parameters, robots, wayback               |