diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index ab2c275a3..66903333d 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -83,6 +83,7 @@ - [jxdv](https://github.com/jxdv) - [Xeonacid](https://github.com/Xeonacid) - [Valentijn Scholten](https://www.github.com/valentijnscholten) +- [partoneplay](https://github.com/partoneplay) Special thanks to all the people who are named here! diff --git a/config.ini b/config.ini index db59de78b..667990dcd 100644 --- a/config.ini +++ b/config.ini @@ -14,6 +14,7 @@ exclude-subdirs = %%ff/,.;/,..;/,;/,./,../,%%2e/,%%2e%%2e/ random-user-agents = False max-time = 0 exit-on-error = False +skip-on-status = 429 #filter-threshold = 10 #subdirs = /,api/ #include-status = 200-299,401 @@ -26,7 +27,6 @@ exit-on-error = False #exclude-regex = "^403$" #exclude-redirect = "*/error.html" #exclude-response = 404.html -#skip-on-status = 429,999 [dictionary] default-extensions = php,asp,aspx,jsp,html,htm diff --git a/db/dicc.txt b/db/dicc.txt index 73a333c9d..88c248c0e 100644 --- a/db/dicc.txt +++ b/db/dicc.txt @@ -3047,14 +3047,21 @@ api/apidocs/swagger.json api/application.wadl api/batch api/cask/graphql +api/chat api/config api/config.json +api/copy +api/create api/credential.json api/credentials.json api/database.json +api/delete api/docs api/docs/ +api/embed +api/embeddings api/error_log +api/generate api/index.html api/jsonws api/jsonws/invoke @@ -3062,6 +3069,10 @@ api/login.json api/package_search/v4/documentation api/profile api/proxy +api/ps +api/pull +api/push +api/show api/snapshots api/spec/swagger.json api/swagger @@ -3073,6 +3084,7 @@ api/swagger/index.html api/swagger/static/index.html api/swagger/swagger api/swagger/ui/index +api/tags api/timelion/run api/user.json api/users.json @@ -8928,9 +8940,19 @@ v1.0/ v1.1 v1/ v1/api-docs +v1/audio/speech +v1/batches +v1/chat/completions +v1/embeddings +v1/files +v1/fine_tuning/jobs +v1/images/generations +v1/models +v1/moderations v1/public/yql v1/test/js/console.html v1/test/js/console_ajax.js +v1/uploads v2 v2.0 v2/ diff --git a/dirsearch.py b/dirsearch.py index dcd539b69..97635d60f 100755 --- a/dirsearch.py +++ b/dirsearch.py @@ -66,6 +66,11 @@ def main(): options.update(parse_options()) + if options["session_file"]: + print("WARNING: Running an untrusted session file might lead to unwanted code execution!") + if input("[c]ontinue / [q]uit: ") != "c": + exit(1) + from lib.controller.controller import Controller Controller() diff --git a/lib/connection/requester.py b/lib/connection/requester.py index c67428346..c72465f04 100755 --- a/lib/connection/requester.py +++ b/lib/connection/requester.py @@ -101,21 +101,6 @@ def set_url(self, url: str) -> None: def set_header(self, key: str, value: str) -> None: self.headers[key] = value.lstrip() - def set_proxy(self, proxy: str) -> None: - if not proxy: - return - - if not proxy.startswith(PROXY_SCHEMES): - proxy = f"http://{proxy}" - - if self.proxy_cred and "@" not in proxy: - # socks5://localhost:9050 => socks5://[credential]@localhost:9050 - proxy = proxy.replace("://", f"://{self.proxy_cred}@", 1) - - self.session.proxies = {"https": proxy} - if not proxy.startswith("https://"): - self.session.proxies["http"] = proxy - def is_rate_exceeded(self) -> bool: return self._rate >= options["max_rate"] > 0 @@ -188,16 +173,24 @@ def request(self, path: str, proxy: str | None = None) -> Response: self.increase_rate() err_msg = None - - # Safe quote all special characters to prevent them from being encoded - url = safequote(self._url + path if self._url else path) + url = self._url + safequote(path) # Why using a loop instead of max_retries argument? Check issue #1009 for _ in range(options["max_retries"] + 1): try: + proxies = {} try: - proxy = proxy or random.choice(options["proxies"]) - self.set_proxy(proxy) + proxy_url = proxy or random.choice(options["proxies"]) + if not proxy_url.startswith(PROXY_SCHEMES): + proxy_url = f"http://{proxy_url}" + + if self.proxy_cred and "@" not in proxy_url: + # socks5://localhost:9050 => socks5://[credential]@localhost:9050 + proxy_url = proxy_url.replace("://", f"://{self.proxy_cred}@", 1) + + proxies["https"] = proxy_url + if not proxy_url.startswith("https://"): + proxies["http"] = proxy_url except IndexError: pass @@ -212,16 +205,17 @@ def request(self, path: str, proxy: str | None = None) -> Response: headers=self.headers, data=options["data"], ) - prepped = self.session.prepare_request(request) - prepped.url = url + prep = self.session.prepare_request(request) + prep.url = url origin_response = self.session.send( - prepped, + prep, allow_redirects=options["follow_redirects"], timeout=options["timeout"], + proxies=proxies, stream=True, ) - response = Response(origin_response) + response = Response(url, origin_response) log_msg = f'"{options["http_method"]} {response.url}" {response.status} - {response.length}B' @@ -359,11 +353,11 @@ async def replay_request(self, path: str, proxy: str) -> AsyncResponse: mounts={"all://": transport}, timeout=httpx.Timeout(options["timeout"]), ) - return await self.request(path, self.replay_session) + return await self.request(path, self.replay_session, replay=True) # :path: is expected not to start with "/" async def request( - self, path: str, session: httpx.AsyncClient | None = None + self, path: str, session: httpx.AsyncClient | None = None, replay: bool = False ) -> AsyncResponse: while self.is_rate_exceeded(): await asyncio.sleep(0.1) @@ -371,12 +365,9 @@ async def request( self.increase_rate() err_msg = None - - # Safe quote all special characters to prevent them from being encoded - url = safequote(self._url + path if self._url else path) - parsed_url = urlparse(url) - + url = self._url + safequote(path) session = session or self.session + for _ in range(options["max_retries"] + 1): try: if self.agents: @@ -388,16 +379,15 @@ async def request( url, headers=self.headers, data=options["data"], + extensions={"target": (url if replay else f"/{safequote(path)}").encode()}, ) - if p := parsed_url.path: - request.extensions = {"target": p.encode()} xresponse = await session.send( request, stream=True, follow_redirects=options["follow_redirects"], ) - response = await AsyncResponse.create(xresponse) + response = await AsyncResponse.create(url, xresponse) await xresponse.aclose() log_msg = f'"{options["http_method"]} {response.url}" {response.status} - {response.length}B' diff --git a/lib/connection/response.py b/lib/connection/response.py index b344f7620..108fe6db0 100755 --- a/lib/connection/response.py +++ b/lib/connection/response.py @@ -35,9 +35,9 @@ class BaseResponse: - def __init__(self, response: requests.Response | httpx.Response) -> None: + def __init__(self, url, response: requests.Response | httpx.Response) -> None: self.datetime = time.strftime("%Y-%m-%d %H:%M:%S") - self.url = str(response.url) + self.url = url self.full_path = parse_path(self.url) self.path = clean_path(self.full_path) self.status = response.status_code @@ -77,8 +77,8 @@ def __eq__(self, other: Any) -> bool: class Response(BaseResponse): - def __init__(self, response: requests.Response) -> None: - super().__init__(response) + def __init__(self, url, response: requests.Response) -> None: + super().__init__(url, response) for chunk in response.iter_content(chunk_size=ITER_CHUNK_SIZE): self.body += chunk @@ -99,8 +99,8 @@ def __init__(self, response: requests.Response) -> None: class AsyncResponse(BaseResponse): @classmethod - async def create(cls, response: httpx.Response) -> AsyncResponse: - self = cls(response) + async def create(cls, url, response: httpx.Response) -> AsyncResponse: + self = cls(url, response) async for chunk in response.aiter_bytes(chunk_size=ITER_CHUNK_SIZE): self.body += chunk diff --git a/lib/controller/controller.py b/lib/controller/controller.py index 88be4dc57..a98e06835 100755 --- a/lib/controller/controller.py +++ b/lib/controller/controller.py @@ -72,11 +72,6 @@ class Controller: def __init__(self) -> None: if options["session_file"]: - print("WARNING: Running an untrusted session file might lead to unwanted code execution!") - interface.in_line("[c]continue / [q]uit: ") - if input() != "c": - exit(1) - self._import(options["session_file"]) self.old_session = True else: @@ -205,12 +200,9 @@ def run(self) -> None: self.requester = Requester() if options["async_mode"]: self.loop = asyncio.new_event_loop() - try: - self.loop.add_signal_handler(signal.SIGINT, self.handle_pause) - except NotImplementedError: - # Windows - signal.signal(signal.SIGINT, self.handle_pause) - signal.signal(signal.SIGTERM, self.handle_pause) + + signal.signal(signal.SIGINT, lambda *_: self.handle_pause()) + signal.signal(signal.SIGTERM, lambda *_: self.handle_pause()) while options["urls"]: url = options["urls"][0] @@ -514,18 +506,14 @@ def is_timed_out(self) -> bool: def process(self) -> None: while True: - try: - while not self.fuzzer.is_finished(): - if self.is_timed_out(): - raise SkipTargetInterrupt( - "Runtime exceeded the maximum set by the user" - ) - time.sleep(0.5) - - break + while not self.fuzzer.is_finished(): + if self.is_timed_out(): + raise SkipTargetInterrupt( + "Runtime exceeded the maximum set by the user" + ) + time.sleep(0.5) - except KeyboardInterrupt: - self.handle_pause() + break def add_directory(self, path: str) -> None: """Add directory to the recursion queue""" diff --git a/lib/core/fuzzer.py b/lib/core/fuzzer.py index dd601223e..ed9577ce0 100755 --- a/lib/core/fuzzer.py +++ b/lib/core/fuzzer.py @@ -154,6 +154,7 @@ def __init__( not_found_callbacks=not_found_callbacks, error_callbacks=error_callbacks, ) + self._exc: Exception | None = None self._threads = [] self._play_event = threading.Event() self._quit_event = threading.Event() @@ -208,13 +209,14 @@ def start(self) -> None: self.setup_scanners() self.setup_threads() self.play() + self._quit_event.clear() for thread in self._threads: thread.start() def is_finished(self) -> bool: - if self.exc: - raise self.exc + if self._exc: + raise self._exc for thread in self._threads: if thread.is_alive(): @@ -238,7 +240,12 @@ def quit(self) -> None: def scan(self, path: str) -> None: scanners = self.get_scanners_for(path) - response = self._requester.request(path) + try: + response = self._requester.request(path) + except RequestException as e: + for callback in self.error_callbacks: + callback(e) + return if self.is_excluded(response): for callback in self.not_found_callbacks: @@ -274,11 +281,8 @@ def thread_proc(self) -> None: except StopIteration: break - except RequestException as e: - for callback in self.error_callbacks: - callback(e) - - continue + except Exception as e: + self._exc = e finally: time.sleep(options["delay"]) @@ -370,12 +374,6 @@ async def start(self) -> None: await asyncio.gather(*self._background_tasks) - def is_finished(self) -> bool: - if self.exc: - raise self.exc - - return len(self._background_tasks) == 0 - def play(self) -> None: self._play_event.set() @@ -388,7 +386,12 @@ def quit(self) -> None: async def scan(self, path: str) -> None: scanners = self.get_scanners_for(path) - response = await self._requester.request(path) + try: + response = await self._requester.request(path) + except RequestException as e: + for callback in self.error_callbacks: + callback(e) + return if self.is_excluded(response): for callback in self.not_found_callbacks: @@ -422,8 +425,5 @@ async def task_proc(self) -> None: await self.scan(self._base_path + path) except StopIteration: pass - except RequestException as e: - for callback in self.error_callbacks: - callback(e) finally: await asyncio.sleep(options["delay"]) diff --git a/requirements.txt b/requirements.txt index 01bd4e608..393dfd6ee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ Jinja2>=3.0.0 defusedxml>=0.7.0 pyopenssl>=21.0.0 requests>=2.27.0 -requests_ntlm>=1.1.0 +requests_ntlm>=1.3.0 colorama>=0.4.4 ntlm_auth>=1.5.0 beautifulsoup4>=4.8.0