docs: update API doc groups

Closes: #1293
apify · vdusek · Jul 18, 2025 · Jul 16, 2025 · Jul 17, 2025 · Jul 18, 2025
commit fcfcf627a28145702efc145b105092e16e412d5b
diff --git a/src/crawlee/_autoscaling/autoscaled_pool.py b/src/crawlee/_autoscaling/autoscaled_pool.py
@@ -35,7 +35,7 @@ def __init__(self) -> None:
         self.result: asyncio.Future = asyncio.Future()
 
 
-@docs_group('Classes')
+@docs_group('Autoscaling')
 class AutoscaledPool:
     """Manages a pool of asynchronous resource-intensive tasks that are executed in parallel.
 

diff --git a/src/crawlee/_autoscaling/snapshotter.py b/src/crawlee/_autoscaling/snapshotter.py
@@ -28,7 +28,7 @@
 T = TypeVar('T')
 
 
-@docs_group('Classes')
+@docs_group('Autoscaling')
 class Snapshotter:
     """Monitors and logs system resource usage at predefined intervals for performance optimization.
 

diff --git a/src/crawlee/_autoscaling/system_status.py b/src/crawlee/_autoscaling/system_status.py
@@ -17,7 +17,7 @@
 logger = getLogger(__name__)
 
 
-@docs_group('Classes')
+@docs_group('Autoscaling')
 class SystemStatus:
     """Provides a simple interface for evaluating system resource usage from snapshots collected by `Snapshotter`.
 

diff --git a/src/crawlee/_request.py b/src/crawlee/_request.py
@@ -138,7 +138,7 @@ class RequestOptions(TypedDict):
     no_retry: NotRequired[bool]
 
 
-@docs_group('Data structures')
+@docs_group('Storage data')
 class Request(BaseModel):
     """Represents a request in the Crawlee framework, containing the necessary information for crawling operations.
 

diff --git a/src/crawlee/_service_locator.py b/src/crawlee/_service_locator.py
@@ -12,7 +12,7 @@
     from crawlee.storages._storage_instance_manager import StorageInstanceManager
 
 
-@docs_group('Classes')
+@docs_group('Configuration')
 class ServiceLocator:
     """Service locator for managing the services used by Crawlee.
 

diff --git a/src/crawlee/_types.py b/src/crawlee/_types.py
@@ -65,7 +65,7 @@ def _normalize_headers(headers: Mapping[str, str]) -> dict[str, str]:
     return dict(sorted_headers)
 
 
-@docs_group('Data structures')
+@docs_group('Others')
 class HttpHeaders(RootModel, Mapping[str, str]):
     """A dictionary-like object representing HTTP headers."""
 
@@ -103,7 +103,7 @@ def __len__(self) -> int:
         return len(self.root)
 
 
-@docs_group('Data structures')
+@docs_group('Configuration')
 class ConcurrencySettings:
     """Concurrency settings for AutoscaledPool."""
 
@@ -507,7 +507,7 @@ def __call__(
         """
 
 
-@docs_group('Data structures')
+@docs_group('Others')
 @dataclasses.dataclass
 class PageSnapshot:
     """Snapshot of a crawled page."""
@@ -547,7 +547,7 @@ def __call__(
 
 
 @dataclass(frozen=True)
-@docs_group('Data structures')
+@docs_group('Crawling contexts')
 class BasicCrawlingContext:
     """Basic crawling context.
 

diff --git a/src/crawlee/_utils/docs.py b/src/crawlee/_utils/docs.py
@@ -3,7 +3,26 @@
 from collections.abc import Callable
 from typing import Any, Literal, TypeVar
 
-GroupName = Literal['Classes', 'Abstract classes', 'Data structures', 'Event payloads', 'Errors', 'Functions']
+GroupName = Literal[
+    'Autoscaling',
+    'Browser management',
+    'Configuration',
+    'Crawlers',
+    'Crawling contexts',
+    'Errors',
+    'Event data',
+    'Event managers',
+    'Functions',
+    'HTTP clients',
+    'HTTP parsers',
+    'Others',
+    'Request loaders',
+    'Session management',
+    'Statistics',
+    'Storage clients',
+    'Storage data',
+    'Storages',
+]
 
 T = TypeVar('T', bound=Callable[..., Any])
 

diff --git a/src/crawlee/browsers/_browser_controller.py b/src/crawlee/browsers/_browser_controller.py
@@ -5,6 +5,8 @@
 from abc import ABC, abstractmethod
 from typing import TYPE_CHECKING, Any
 
+from crawlee._utils.docs import docs_group
+
 if TYPE_CHECKING:
     from collections.abc import Mapping
     from datetime import datetime, timedelta
@@ -15,6 +17,7 @@
     from crawlee.proxy_configuration import ProxyInfo
 
 
+@docs_group('Browser management')
 class BrowserController(ABC):
     """An abstract base class for managing browser instance and their pages."""
 

diff --git a/src/crawlee/browsers/_browser_plugin.py b/src/crawlee/browsers/_browser_plugin.py
@@ -5,6 +5,8 @@
 from abc import ABC, abstractmethod
 from typing import TYPE_CHECKING, Any
 
+from crawlee._utils.docs import docs_group
+
 if TYPE_CHECKING:
     from collections.abc import Mapping
     from types import TracebackType
@@ -13,6 +15,7 @@
     from crawlee.browsers._types import BrowserType
 
 
+@docs_group('Browser management')
 class BrowserPlugin(ABC):
     """An abstract base class for browser plugins.
 

diff --git a/src/crawlee/browsers/_browser_pool.py b/src/crawlee/browsers/_browser_pool.py
@@ -30,7 +30,7 @@
 logger = getLogger(__name__)
 
 
-@docs_group('Classes')
+@docs_group('Browser management')
 class BrowserPool:
     """Manage a pool of browsers and pages, handling their lifecycle and resource allocation.
 

diff --git a/src/crawlee/browsers/_playwright_browser.py b/src/crawlee/browsers/_playwright_browser.py
@@ -10,12 +10,15 @@
 from playwright.async_api import Browser
 from typing_extensions import override
 
+from crawlee._utils.docs import docs_group
+
 if TYPE_CHECKING:
     from playwright.async_api import BrowserContext, BrowserType, CDPSession, Page
 
 logger = getLogger(__name__)
 
 
+@docs_group('Browser management')
 class PlaywrightPersistentBrowser(Browser):
     """A wrapper for Playwright's `Browser` that operates with a persistent context.
 

diff --git a/src/crawlee/browsers/_playwright_browser_controller.py b/src/crawlee/browsers/_playwright_browser_controller.py
@@ -27,7 +27,7 @@
 logger = getLogger(__name__)
 
 
-@docs_group('Classes')
+@docs_group('Browser management')
 class PlaywrightBrowserController(BrowserController):
     """Controller for managing Playwright browser instances and their pages.
 

diff --git a/src/crawlee/browsers/_playwright_browser_plugin.py b/src/crawlee/browsers/_playwright_browser_plugin.py
@@ -28,7 +28,7 @@
 logger = getLogger(__name__)
 
 
-@docs_group('Classes')
+@docs_group('Browser management')
 class PlaywrightBrowserPlugin(BrowserPlugin):
     """A plugin for managing Playwright automation library.
 

diff --git a/src/crawlee/configuration.py b/src/crawlee/configuration.py
@@ -16,7 +16,7 @@
 __all__ = ['Configuration']
 
 
-@docs_group('Data structures')
+@docs_group('Configuration')
 class Configuration(BaseSettings):
     """Configuration settings for the Crawlee project.
 

diff --git a/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py b/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py
@@ -32,7 +32,7 @@
 TStatisticsState = TypeVar('TStatisticsState', bound=StatisticsState, default=StatisticsState)
 
 
-@docs_group('Abstract classes')
+@docs_group('Crawlers')
 class AbstractHttpCrawler(
     Generic[TCrawlingContext, TParseResult, TSelectResult], BasicCrawler[TCrawlingContext, StatisticsState], ABC
 ):

diff --git a/src/crawlee/crawlers/_abstract_http/_abstract_http_parser.py b/src/crawlee/crawlers/_abstract_http/_abstract_http_parser.py
@@ -15,13 +15,13 @@
     from crawlee.http_clients import HttpResponse
 
 
-@docs_group('Abstract classes')
+@docs_group('HTTP parsers')
 class AbstractHttpParser(Generic[TParseResult, TSelectResult], ABC):
-    """Parser used for parsing http response and inspecting parsed result to find links or detect blocking."""
+    """Parser used for parsing HTTP response and inspecting parsed result to find links or detect blocking."""
 
     @abstractmethod
     async def parse(self, response: HttpResponse) -> TParseResult:
-        """Parse http response.
+        """Parse HTTP response.
 
         Args:
             response: HTTP response to be parsed.

diff --git a/src/crawlee/crawlers/_abstract_http/_http_crawling_context.py b/src/crawlee/crawlers/_abstract_http/_http_crawling_context.py
@@ -14,7 +14,7 @@
 
 
 @dataclass(frozen=True)
-@docs_group('Data structures')
+@docs_group('Crawling contexts')
 class HttpCrawlingContext(BasicCrawlingContext, HttpCrawlingResult):
     """The crawling context used by the `AbstractHttpCrawler`."""
 
@@ -30,7 +30,7 @@ async def get_snapshot(self) -> PageSnapshot:
 
 
 @dataclass(frozen=True)
-@docs_group('Data structures')
+@docs_group('Crawling contexts')
 class ParsedHttpCrawlingContext(Generic[TParseResult], HttpCrawlingContext):
     """The crawling context used by `AbstractHttpCrawler`.
 

diff --git a/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py b/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py
@@ -83,7 +83,7 @@ async def __aexit__(
         self._active = False
 
 
-@docs_group('Classes')
+@docs_group('Crawlers')
 class AdaptivePlaywrightCrawler(
     Generic[TStaticCrawlingContext, TStaticParseResult, TStaticSelectResult],
     BasicCrawler[AdaptivePlaywrightCrawlingContext, AdaptivePlaywrightCrawlerStatisticState],

diff --git a/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler_statistics.py b/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler_statistics.py
@@ -8,7 +8,7 @@
 from crawlee.statistics import StatisticsState
 
 
-@docs_group('Data structures')
+@docs_group('Statistics')
 class AdaptivePlaywrightCrawlerStatisticState(StatisticsState):
     """Statistic data about a crawler run with additional information related to adaptive crawling."""
 

diff --git a/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawling_context.py b/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawling_context.py
@@ -29,7 +29,7 @@ class AdaptiveContextError(RuntimeError):
 
 
 @dataclass(frozen=True)
-@docs_group('Data structures')
+@docs_group('Crawling contexts')
 class AdaptivePlaywrightCrawlingContext(
     Generic[TStaticParseResult, TStaticSelectResult], ParsedHttpCrawlingContext[TStaticParseResult]
 ):
@@ -200,7 +200,7 @@ async def from_playwright_crawling_context(
 
 
 @dataclass(frozen=True)
-@docs_group('Data structures')
+@docs_group('Crawling contexts')
 class AdaptivePlaywrightPreNavCrawlingContext(BasicCrawlingContext):
     """A wrapper around BasicCrawlingContext or AdaptivePlaywrightCrawlingContext.
 

diff --git a/src/crawlee/crawlers/_adaptive_playwright/_rendering_type_predictor.py b/src/crawlee/crawlers/_adaptive_playwright/_rendering_type_predictor.py
@@ -18,7 +18,7 @@
 FeatureVector = tuple[float, float]
 
 
-@docs_group('Data structures')
+@docs_group('Others')
 @dataclass(frozen=True)
 class RenderingTypePrediction:
     """Rendering type recommendation with detection probability recommendation."""
@@ -32,7 +32,7 @@ class RenderingTypePrediction:
     One represents no confidence in `rendering_type` recommendation."""
 
 
-@docs_group('Classes')
+@docs_group('Others')
 class RenderingTypePredictor(ABC):
     """Stores rendering type for previously crawled URLs and predicts the rendering type for unvisited urls."""
 
@@ -54,7 +54,7 @@ def store_result(self, request: Request, rendering_type: RenderingType) -> None:
         """
 
 
-@docs_group('Classes')
+@docs_group('Others')
 class DefaultRenderingTypePredictor(RenderingTypePredictor):
     """Stores rendering type for previously crawled URLs and predicts the rendering type for unvisited urls.
 

diff --git a/src/crawlee/crawlers/_basic/_basic_crawler.py b/src/crawlee/crawlers/_basic/_basic_crawler.py
@@ -218,7 +218,6 @@ class _BasicCrawlerOptionsGeneric(Generic[TCrawlingContext, TStatisticsState], T
     """A custom `Statistics` instance, allowing the use of non-default configuration."""
 
 
-@docs_group('Data structures')
 class BasicCrawlerOptions(
     Generic[TCrawlingContext, TStatisticsState],
     _BasicCrawlerOptions,
@@ -230,7 +229,7 @@ class BasicCrawlerOptions(
     """
 
 
-@docs_group('Classes')
+@docs_group('Crawlers')
 class BasicCrawler(Generic[TCrawlingContext, TStatisticsState]):
     """A basic web crawler providing a framework for crawling websites.
 

diff --git a/src/crawlee/crawlers/_basic/_context_pipeline.py b/src/crawlee/crawlers/_basic/_context_pipeline.py
@@ -53,7 +53,7 @@ async def cleanup(self, final_consumer_exception: Exception | None) -> None:
             raise RuntimeError('The middleware yielded more than once')
 
 
-@docs_group('Classes')
+@docs_group('Others')
 class ContextPipeline(Generic[TCrawlingContext]):
     """Encapsulates the logic of gradually enhancing the crawling context with additional information and utilities.
 

diff --git a/src/crawlee/crawlers/_beautifulsoup/_beautifulsoup_crawler.py b/src/crawlee/crawlers/_beautifulsoup/_beautifulsoup_crawler.py
@@ -18,7 +18,7 @@
     from crawlee.crawlers._abstract_http import ParsedHttpCrawlingContext
 
 
-@docs_group('Classes')
+@docs_group('Crawlers')
 class BeautifulSoupCrawler(AbstractHttpCrawler[BeautifulSoupCrawlingContext, BeautifulSoup, Tag]):
     """A web crawler for performing HTTP requests and parsing HTML/XML content.
 

diff --git a/src/crawlee/crawlers/_beautifulsoup/_beautifulsoup_crawling_context.py b/src/crawlee/crawlers/_beautifulsoup/_beautifulsoup_crawling_context.py
@@ -10,7 +10,7 @@
 
 
 @dataclass(frozen=True)
-@docs_group('Data structures')
+@docs_group('Crawling contexts')
 class BeautifulSoupCrawlingContext(ParsedHttpCrawlingContext[BeautifulSoup]):
     """The crawling context used by the `BeautifulSoupCrawler`.
 

diff --git a/src/crawlee/crawlers/_beautifulsoup/_beautifulsoup_parser.py b/src/crawlee/crawlers/_beautifulsoup/_beautifulsoup_parser.py
@@ -5,6 +5,7 @@
 from bs4 import BeautifulSoup, Tag
 from typing_extensions import override
 
+from crawlee._utils.docs import docs_group
 from crawlee.crawlers._abstract_http import AbstractHttpParser
 
 if TYPE_CHECKING:
@@ -13,6 +14,7 @@
     from crawlee.http_clients import HttpResponse
 
 
+@docs_group('HTTP parsers')
 class BeautifulSoupParser(AbstractHttpParser[BeautifulSoup, Tag]):
     """Parser for parsing HTTP response using `BeautifulSoup`."""
 

diff --git a/src/crawlee/crawlers/_http/_http_crawler.py b/src/crawlee/crawlers/_http/_http_crawler.py
@@ -13,7 +13,7 @@
     from crawlee.crawlers import BasicCrawlerOptions
 
 
-@docs_group('Classes')
+@docs_group('Crawlers')
 class HttpCrawler(AbstractHttpCrawler[ParsedHttpCrawlingContext[bytes], bytes, bytes]):
     """Specific version of generic `AbstractHttpCrawler`.
 

diff --git a/src/crawlee/crawlers/_http/_http_parser.py b/src/crawlee/crawlers/_http/_http_parser.py
@@ -4,6 +4,7 @@
 
 from typing_extensions import override
 
+from crawlee._utils.docs import docs_group
 from crawlee.crawlers._abstract_http import AbstractHttpParser
 from crawlee.crawlers._types import BlockedInfo
 
@@ -13,10 +14,12 @@
     from crawlee.http_clients import HttpResponse
 
 
+@docs_group('HTTP parsers')
 class NoParser(AbstractHttpParser[bytes, bytes]):
-    """Dummy parser for backwards compatibility.
+    """A no-op parser that returns raw response content without any processing.
 
-    To enable using `HttpCrawler` without need for additional specific parser.
+    This is useful when you only need the raw response data and don't require HTML
+    parsing, link extraction, or content selection functionality.
     """
 
     @override