Validate postman workspaces

bbot/modules/base.py

@@ -1176,10 +1176,17 @@ async def api_request(self, *args, **kwargs):
 
         return r
 
+    def _prepare_api_iter_req(self, url, page, page_size, offset, **requests_kwargs):
+        """
+        Default function for preparing an API request for iterating through paginated data.
+        """
+        url = self.helpers.safe_format(url, page=page, page_size=page_size, offset=offset)
+        return url, requests_kwargs
+
     def _api_response_is_success(self, r):
         return r.is_success
 
-    async def api_page_iter(self, url, page_size=100, json=True, next_key=None, **requests_kwargs):
+    async def api_page_iter(self, url, page_size=100, _json=True, next_key=None, iter_key=None, **requests_kwargs):
         """
         An asynchronous generator function for iterating through paginated API data.
 
@@ -1192,6 +1199,7 @@ async def api_page_iter(self, url, page_size=100, json=True, next_key=None, **re
             page_size (int, optional): The number of items per page. Defaults to 100.
             json (bool, optional): If True, attempts to deserialize the response content to a JSON object. Defaults to True.
             next_key (callable, optional): A function that takes the last page's data and returns the URL for the next page. Defaults to None.
+            iter_key (callable, optional): A function that builds each new request based on the page number, page size, and offset. Defaults to a simple implementation that autoreplaces {page} and {page_size} in the url.
             **requests_kwargs: Arbitrary keyword arguments that will be forwarded to the HTTP request function.
 
         Yields:
@@ -1214,6 +1222,8 @@ async def api_page_iter(self, url, page_size=100, json=True, next_key=None, **re
         page = 1
         offset = 0
         result = None
+        if iter_key is None:
+            iter_key = self._prepare_api_iter_req
         while 1:
             if result and callable(next_key):
                 try:
@@ -1222,13 +1232,13 @@ async def api_page_iter(self, url, page_size=100, json=True, next_key=None, **re
                     self.debug(f"Failed to extract next page of results from {url}: {e}")
                     self.debug(traceback.format_exc())
             else:
-                new_url = self.helpers.safe_format(url, page=page, page_size=page_size, offset=offset)
-            result = await self.api_request(new_url, **requests_kwargs)
+                new_url, new_kwargs = iter_key(url, page, page_size, offset, **requests_kwargs)
+            result = await self.api_request(new_url, **new_kwargs)
             if result is None:
                 self.verbose(f"api_page_iter() got no response for {url}")
                 break
             try:
-                if json:
+                if _json:
                     result = result.json()
                 yield result
             except Exception:

bbot/modules/dehashed.py

@@ -90,7 +90,7 @@ async def query(self, domain):
         url = f"{self.base_url}?query={query}&size=10000&page=" + "{page}"
         page = 0
         num_entries = 0
-        agen = self.api_page_iter(url=url, auth=self.auth, headers=self.headers, json=False)
+        agen = self.api_page_iter(url=url, auth=self.auth, headers=self.headers, _json=False)
         async for result in agen:
             result_json = {}
             with suppress(Exception):

bbot/modules/dockerhub.py

@@ -64,7 +64,7 @@ async def handle_social(self, event):
     async def get_repos(self, username):
         repos = []
         url = f"{self.api_url}/repositories/{username}?page_size=25&page=" + "{page}"
-        agen = self.api_page_iter(url, json=False)
+        agen = self.api_page_iter(url, _json=False)
         try:
             async for r in agen:
                 if r is None:

bbot/modules/github_codesearch.py

@@ -42,7 +42,7 @@ async def handle_event(self, event):
     async def query(self, query):
         repos = {}
         url = f"{self.base_url}/search/code?per_page=100&type=Code&q={self.helpers.quote(query)}&page=" + "{page}"
-        agen = self.api_page_iter(url, headers=self.headers, json=False)
+        agen = self.api_page_iter(url, headers=self.headers, _json=False)
         num_results = 0
         try:
             async for r in agen:

bbot/modules/github_org.py

@@ -108,7 +108,7 @@ async def handle_event(self, event):
     async def query_org_repos(self, query):
         repos = []
         url = f"{self.base_url}/orgs/{self.helpers.quote(query)}/repos?per_page=100&page=" + "{page}"
-        agen = self.api_page_iter(url, json=False)
+        agen = self.api_page_iter(url, _json=False)
         try:
             async for r in agen:
                 if r is None:
@@ -136,7 +136,7 @@ async def query_org_repos(self, query):
     async def query_org_members(self, query):
         members = []
         url = f"{self.base_url}/orgs/{self.helpers.quote(query)}/members?per_page=100&page=" + "{page}"
-        agen = self.api_page_iter(url, json=False)
+        agen = self.api_page_iter(url, _json=False)
         try:
             async for r in agen:
                 if r is None:
@@ -164,7 +164,7 @@ async def query_org_members(self, query):
     async def query_user_repos(self, query):
         repos = []
         url = f"{self.base_url}/users/{self.helpers.quote(query)}/repos?per_page=100&page=" + "{page}"
-        agen = self.api_page_iter(url, json=False)
+        agen = self.api_page_iter(url, _json=False)
         try:
             async for r in agen:
                 if r is None:

bbot/modules/github_workflows.py

@@ -92,7 +92,7 @@ async def handle_event(self, event):
     async def get_workflows(self, owner, repo):
         workflows = []
         url = f"{self.base_url}/repos/{owner}/{repo}/actions/workflows?per_page=100&page=" + "{page}"
-        agen = self.api_page_iter(url, json=False)
+        agen = self.api_page_iter(url, _json=False)
         try:
             async for r in agen:
                 if r is None:

bbot/modules/postman.py

@@ -10,52 +10,64 @@ class postman(postman):
         "created_date": "2024-09-07",
         "author": "@domwhewell-sage",
     }
-
+    options = {"api_key": ""}
+    options_desc = {"api_key": "Postman API Key"}
     reject_wildcards = False
 
     async def handle_event(self, event):
         # Handle postman profile
         if event.type == "SOCIAL":
-            await self.handle_profile(event)
+            owner = event.data.get("profile_name", "")
+            in_scope_workspaces = await self.process_workspaces(user=owner)
         elif event.type == "ORG_STUB":
-            await self.handle_org_stub(event)
-
-    async def handle_profile(self, event):
-        profile_name = event.data.get("profile_name", "")
-        self.verbose(f"Searching for postman workspaces, collections, requests belonging to {profile_name}")
-        for item in await self.query(profile_name):
-            workspace = item["document"]
-            name = workspace["slug"]
-            profile = workspace["publisherHandle"]
-            if profile_name.lower() == profile.lower():
-                self.verbose(f"Got {name}")
-                workspace_url = f"{self.html_url}/{profile}/{name}"
+            owner = event.data
+            in_scope_workspaces = await self.process_workspaces(org=owner)
+        if in_scope_workspaces:
+            for workspace in in_scope_workspaces:
+                repo_url = workspace["url"]
+                repo_name = workspace["repo_name"]
+                if event.type == "SOCIAL":
+                    context = f'{{module}} searched postman.com for workspaces belonging to "{owner}" and found "{repo_name}" at {{event.type}}: {repo_url}'
+                elif event.type == "ORG_STUB":
+                    context = f'{{module}} searched postman.com for "{owner}" and found matching workspace "{repo_name}" at {{event.type}}: {repo_url}'
                 await self.emit_event(
-                    {"url": workspace_url},
+                    {"url": repo_url},
                     "CODE_REPOSITORY",
                     tags="postman",
                     parent=event,
-                    context=f'{{module}} searched postman.com for workspaces belonging to "{profile_name}" and found "{name}" at {{event.type}}: {workspace_url}',
+                    context=context,
                 )
 
-    async def handle_org_stub(self, event):
-        org_name = event.data
-        self.verbose(f"Searching for any postman workspaces, collections, requests for {org_name}")
-        for item in await self.query(org_name):
-            workspace = item["document"]
-            name = workspace["slug"]
-            profile = workspace["publisherHandle"]
-            self.verbose(f"Got {name}")
-            workspace_url = f"{self.html_url}/{profile}/{name}"
-            await self.emit_event(
-                {"url": workspace_url},
-                "CODE_REPOSITORY",
-                tags="postman",
-                parent=event,
-                context=f'{{module}} searched postman.com for "{org_name}" and found matching workspace "{name}" at {{event.type}}: {workspace_url}',
-            )
+    async def process_workspaces(self, user=None, org=None):
+        in_scope_workspaces = []
+        owner = user or org
+        if owner:
+            self.verbose(f"Searching for postman workspaces, collections, requests for {owner}")
+            for item in await self.query(owner):
+                workspace = item["document"]
+                slug = workspace["slug"]
+                profile = workspace["publisherHandle"]
+                repo_url = f"{self.html_url}/{profile}/{slug}"
+                workspace_id = await self.get_workspace_id(repo_url)
+                if (org and workspace_id) or (user and owner.lower() == profile.lower()):
+                    self.verbose(f"Found workspace ID {workspace_id} for {repo_url}")
+                    data = await self.request_workspace(workspace_id)
+                    in_scope = await self.validate_workspace(
+                        data["workspace"], data["environments"], data["collections"]
+                    )
+                    if in_scope:
+                        in_scope_workspaces.append({"url": repo_url, "repo_name": slug})
+                    else:
+                        self.verbose(
+                            f"Failed to validate {repo_url} is in our scope as it does not contain any in-scope dns_names / emails"
+                        )
+        return in_scope_workspaces
 
     async def query(self, query):
+        def api_page_iter(url, page, page_size, offset, **kwargs):
+            kwargs["json"]["body"]["from"] = offset
+            return url, kwargs
+
         data = []
         url = f"{self.base_url}/ws/proxy"
         json = {
@@ -67,7 +79,7 @@ async def query(self, query):
                     "collaboration.workspace",
                 ],
                 "queryText": self.helpers.quote(query),
-                "size": 100,
+                "size": 25,
                 "from": 0,
                 "clientTraceId": "",
                 "requestOrigin": "srp",
@@ -76,13 +88,19 @@ async def query(self, query):
                 "domain": "public",
             },
         }
-        r = await self.helpers.request(url, method="POST", json=json, headers=self.headers)
-        if r is None:
-            return data
-        status_code = getattr(r, "status_code", 0)
-        try:
-            json = r.json()
-        except Exception as e:
-            self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}")
-            return None
-        return json.get("data", [])
+
+        agen = self.api_page_iter(
+            url, page_size=25, method="POST", iter_key=api_page_iter, json=json, _json=False, headers=self.headers
+        )
+        async for r in agen:
+            status_code = getattr(r, "status_code", 0)
+            if status_code != 200:
+                self.debug(f"Reached end of postman search results (url: {r.url}) with status code {status_code}")
+                break
+            try:
+                data.extend(r.json().get("data", []))
+            except Exception as e:
+                self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}")
+                return None
+
+        return data