diff --git a/bbot/modules/trufflehog.py b/bbot/modules/trufflehog.py index 9254a63132..3b54b01166 100644 --- a/bbot/modules/trufflehog.py +++ b/bbot/modules/trufflehog.py @@ -37,6 +37,7 @@ class Config(BaseModuleConfig): ] scope_distance_modifier = 2 + _module_threads = 2 async def setup_deps(self): self.config_file = self.config.get("config", "") @@ -86,6 +87,8 @@ async def handle_event(self, event): if isinstance(event.data, dict): description = event.data.get("description", "") + path = None + stdin_data = None if event.type == "CODE_REPOSITORY": path = event.url module = "github-experimental" @@ -100,12 +103,8 @@ async def handle_event(self, event): else: module = "filesystem" elif event.type in ("HTTP_RESPONSE", "RAW_TEXT"): - module = "filesystem" - file_data = event.raw_response if event.type == "HTTP_RESPONSE" else event.data - # write the response to a tempfile - # this is necessary because trufflehog doesn't yet support reading from stdin - # https://github.com/trufflesecurity/trufflehog/issues/162 - path = self.helpers.tempfile(file_data, pipe=False) + module = "stdin" + stdin_data = event.raw_response if event.type == "HTTP_RESPONSE" else event.data if event.type == "CODE_REPOSITORY": host = event.host @@ -118,7 +117,7 @@ async def handle_event(self, event): rawv2_result, verified, source_metadata, - ) in self.execute_trufflehog(module, path): + ) in self.execute_trufflehog(module, path=path, stdin_data=stdin_data): verified_str = "Verified" if verified else "Possible" confidence = "CONFIRMED" if verified else "MEDIUM" data = { @@ -127,6 +126,10 @@ async def handle_event(self, event): } if host: data["host"] = host + if event.type == "HTTP_RESPONSE": + url = event.data.get("url", "") + if url: + data["url"] = url data["severity"] = "HIGH" data["confidence"] = confidence @@ -142,11 +145,7 @@ async def handle_event(self, event): context=f'{{module}} searched {event.type} using "{module}" method and found {verified_str.lower()} secret ({{event.type}}): {raw_result}', ) - # clean up the tempfile when we're done with it - if event.type in ("HTTP_RESPONSE", "RAW_TEXT"): - path.unlink(missing_ok=True) - - async def execute_trufflehog(self, module, path=None, string=None): + async def execute_trufflehog(self, module, path=None, stdin_data=None): command = [ "trufflehog", "--json", @@ -169,6 +168,8 @@ async def execute_trufflehog(self, module, path=None, string=None): elif module == "filesystem": command.append("filesystem") command.append(path) + elif module == "stdin": + command.append("stdin") elif module == "github-experimental": command.append("github-experimental") command.append("--repo=" + path) @@ -176,10 +177,14 @@ async def execute_trufflehog(self, module, path=None, string=None): command.append("--delete-cached-data") command.append("--token=" + self.github_token) - stats_file = self.helpers.tempfile_tail(callback=partial(self.log_trufflehog_status, path)) + run_kwargs = {} + if stdin_data is not None: + run_kwargs["input"] = stdin_data + + stats_file = self.helpers.tempfile_tail(callback=partial(self.log_trufflehog_status, path or module)) try: with open(stats_file, "w") as stats_fh: - async for line in self.helpers.run_live(command, stderr=stats_fh): + async for line in self.run_process_live(command, stderr=stats_fh, **run_kwargs): try: j = json.loads(line) except json.decoder.JSONDecodeError: @@ -202,7 +207,7 @@ async def execute_trufflehog(self, module, path=None, string=None): finally: stats_file.unlink(missing_ok=True) - def log_trufflehog_status(self, path, line): + def log_trufflehog_status(self, target, line): try: line = json.loads(line) except Exception: @@ -211,5 +216,5 @@ def log_trufflehog_status(self, path, line): message = line.get("msg", "") ts = line.get("ts", "") status = f"Message: {message} | Timestamp: {ts}" - self.verbose(f"Current scan target: {path}") + self.verbose(f"Current scan target: {target}") self.verbose(status) diff --git a/bbot/presets/web/js-audit.yml b/bbot/presets/web/js-audit.yml new file mode 100644 index 0000000000..a408f40786 --- /dev/null +++ b/bbot/presets/web/js-audit.yml @@ -0,0 +1,20 @@ +description: Hunt for leaked credentials and vulnerable libraries in client-side JavaScript + + +modules: + - http + - trufflehog + - badsecrets + - retirejs + - robots + - wayback + +config: + modules: + trufflehog: + only_verified: false + robots: + include_sitemap: true + wayback: + urls: true + archive: true diff --git a/bbot/test/test_step_2/module_tests/test_module_trufflehog.py b/bbot/test/test_step_2/module_tests/test_module_trufflehog.py index 0e9ba6332a..bafa92d744 100644 --- a/bbot/test/test_step_2/module_tests/test_module_trufflehog.py +++ b/bbot/test/test_step_2/module_tests/test_module_trufflehog.py @@ -1328,7 +1328,11 @@ async def setup_before_prep(self, module_test): module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) def check(self, module_test, events): - assert any(e.type == "FINDING" for e in events) + findings = [e for e in events if e.type == "FINDING"] + assert findings, "trufflehog produced no FINDING for HTTP_RESPONSE with a secret" + assert all(f.data.get("url", "").startswith("http://127.0.0.1:8888") for f in findings), ( + "FINDING must carry the source URL of the HTTP_RESPONSE, not a tempfile path" + ) class TestTrufflehog_RAWText(ModuleTestBase): @@ -1357,3 +1361,35 @@ def check(self, module_test, events): # Trufflehog emits HIGH severity and MEDIUM confidence for possible secrets assert finding_events[0].data["severity"] == "HIGH" assert finding_events[0].data["confidence"] == "MEDIUM" + + +class TestTrufflehog_JSSecretURL(ModuleTestBase): + # A secret in a linked JS bundle must produce a FINDING whose url is the JS URL, + # not the HTML URL and not a tempfile path. + targets = ["http://127.0.0.1:8888"] + modules_overrides = ["http", "excavate", "trufflehog"] + config_overrides = { + "modules": {"trufflehog": {"only_verified": False}}, + "web": {"spider_distance": 1, "spider_depth": 1}, + } + + # Split the webhook literal so the source doesn't match GitHub's push-protection + # secret-scanning regex; assembled at runtime, trufflehog still detects it. + _slack_webhook = "https://hooks.slack.com/services/T7KJ4NLXR/B8QZ" + "2MP3V/xJ9vNqLpZ4dKcYm2XwRfBg7T" + + async def setup_before_prep(self, module_test): + module_test.set_expect_requests( + expect_args={"method": "GET", "uri": "/"}, + respond_args={"response_data": ''}, + ) + module_test.set_expect_requests( + expect_args={"method": "GET", "uri": "/app.js"}, + respond_args={"response_data": f'const w = "{self._slack_webhook}";'}, + ) + + def check(self, module_test, events): + js_findings = [e for e in events if e.type == "FINDING" and "SlackWebhook" in e.data.get("description", "")] + assert js_findings, "trufflehog produced no SlackWebhook FINDING from linked JS" + assert all(f.data.get("url", "") == "http://127.0.0.1:8888/app.js" for f in js_findings), ( + f"FINDING must carry the JS URL, got: {[f.data.get('url') for f in js_findings]}" + ) diff --git a/docs/scanning/presets_list.md b/docs/scanning/presets_list.md index 5ba52e80ff..605ffeeb5d 100644 --- a/docs/scanning/presets_list.md +++ b/docs/scanning/presets_list.md @@ -199,6 +199,38 @@ Category: web Modules: [0]("") +## **js-audit** + +Hunt for leaked credentials and vulnerable libraries in client-side JavaScript + +??? note "`js-audit.yml`" + ```yaml title="~/.bbot/presets/web/js-audit.yml" + description: Hunt for leaked credentials and vulnerable libraries in client-side JavaScript + + + modules: + - http + - trufflehog + - badsecrets + - retirejs + - robots + - wayback + + config: + modules: + trufflehog: + only_verified: false + robots: + include_sitemap: true + wayback: + urls: true + archive: true + ``` + +Category: web + +Modules: [0]("") + ## **kitchen-sink** Everything everywhere all at once @@ -991,6 +1023,7 @@ Here is a the same data, but in a table: | email-enum | | Enumerate email addresses from APIs, web crawling, etc. | 0 | | | fast | | Scan only the provided targets as fast as possible - no extra discovery | 0 | | | iis-shortnames | web | Recursively enumerate IIS shortnames | 0 | | +| js-audit | web | Hunt for leaked credentials and vulnerable libraries in client-side JavaScript | 6 | badsecrets, http, retirejs, robots, trufflehog, wayback | | kitchen-sink | | Everything everywhere all at once | 7 | baddns, baddns_direct, baddns_zone, http, hunt, reflected_parameters, webbrute | | lightfuzz | web | Default fuzzing: all 9 submodules (cmdi, crypto, path, serial, sqli, ssti, xss, esi, ssrf) plus companion modules (badsecrets, hunt, reflected_parameters). POST fuzzing disabled but try_post_as_get enabled, so POST params are retested as GET. Skips confirmed WAFs. | 6 | badsecrets, http, hunt, lightfuzz, portfilter, reflected_parameters | | lightfuzz-heavy | web | Aggressive fuzzing: everything in lightfuzz, plus paramminer brute-force parameter discovery (headers, GET params, cookies), POST request fuzzing enabled, try_get_as_post enabled (GET params retested as POST), and robots.txt parsing. Still skips confirmed WAFs. | 8 | badsecrets, http, hunt, lightfuzz, portfilter, reflected_parameters, robots, wayback |