docs: reexport storages documentation from crawlee-python (#328)

Updates the `@apify/docusaurus-plugin-typedoc-api` plugin, which enables
us to use the unified Python documentation pipeline (instead of handling
it in each Python project separately).

Decorates the symbols to be documented with `@docs_group` and
`@docs_name` decorators (just like in
[crawlee-python](apify/crawlee-python#655)).
Those decorators could be moved up into some shared package (e.g.
`apify-shared`, which already has the `@ignore_docs` decorator).

Uses the new `reexports` feature of the plugin to load external API docs
(from `crawlee.dev/python`) and re-render them in the current
documentation instance.

Author: barjin

Makefile

@@ -49,7 +49,7 @@ build-api-reference:
 	cd website && poetry run ./build_api_reference.sh
 
 build-docs:
-	cd website && npm clean-install && npm run build
+	cd website && npm clean-install && poetry run npm run build
 
 run-docs: build-api-reference
-	cd website && npm clean-install && npm run start
+	cd website && npm clean-install && poetry run npm run start

src/apify/_actor.py

@@ -22,7 +22,7 @@
 from apify._models import ActorRun
 from apify._platform_event_manager import EventManager, LocalEventManager, PlatformEventManager
 from apify._proxy_configuration import ProxyConfiguration
-from apify._utils import get_system_info, is_running_in_ipython
+from apify._utils import docs_group, docs_name, get_system_info, is_running_in_ipython
 from apify.apify_storage_client import ApifyStorageClient
 from apify.log import _configure_logging, logger
 from apify.storages import Dataset, KeyValueStore, RequestQueue
@@ -39,6 +39,8 @@
 MainReturnType = TypeVar('MainReturnType')
 
 
+@docs_name('Actor')
+@docs_group('Classes')
 class _ActorType:
     """The class of `Actor`. Only make a new instance if you're absolutely sure you need to."""
 

src/apify/_configuration.py

@@ -11,7 +11,10 @@
 from crawlee._utils.urls import validate_http_url
 from crawlee.configuration import Configuration as CrawleeConfiguration
 
+from apify._utils import docs_group
 
+
+@docs_group('Classes')
 class Configuration(CrawleeConfiguration):
     """A class for specifying the configuration of an Actor.
 

src/apify/_models.py

@@ -10,7 +10,10 @@
 from crawlee._utils.models import timedelta_ms
 from crawlee._utils.urls import validate_http_url
 
+from apify._utils import docs_group
 
+
+@docs_group('Data structures')
 class Webhook(BaseModel):
     __model_config__ = ConfigDict(populate_by_name=True)
 
@@ -29,12 +32,14 @@ class Webhook(BaseModel):
     ] = None
 
 
+@docs_group('Data structures')
 class ActorRunMeta(BaseModel):
     __model_config__ = ConfigDict(populate_by_name=True)
 
     origin: Annotated[MetaOrigin, Field()]
 
 
+@docs_group('Data structures')
 class ActorRunStats(BaseModel):
     __model_config__ = ConfigDict(populate_by_name=True)
 
@@ -55,6 +60,7 @@ class ActorRunStats(BaseModel):
     compute_units: Annotated[float, Field(alias='computeUnits')]
 
 
+@docs_group('Data structures')
 class ActorRunOptions(BaseModel):
     __model_config__ = ConfigDict(populate_by_name=True)
 
@@ -64,6 +70,7 @@ class ActorRunOptions(BaseModel):
     disk_mbytes: Annotated[int, Field(alias='diskMbytes')]
 
 
+@docs_group('Data structures')
 class ActorRunUsage(BaseModel):
     __model_config__ = ConfigDict(populate_by_name=True)
 
@@ -81,6 +88,7 @@ class ActorRunUsage(BaseModel):
     proxy_serps: Annotated[float | None, Field(alias='PROXY_SERPS')] = None
 
 
+@docs_group('Data structures')
 class ActorRun(BaseModel):
     __model_config__ = ConfigDict(populate_by_name=True)
 

src/apify/_platform_event_manager.py

@@ -19,6 +19,7 @@
     EventSystemInfoData,
 )
 
+from apify._utils import docs_group
 from apify.log import logger
 
 if TYPE_CHECKING:
@@ -30,11 +31,13 @@
 __all__ = ['EventManager', 'LocalEventManager', 'PlatformEventManager']
 
 
+@docs_group('Data structures')
 class PersistStateEvent(BaseModel):
     name: Literal[Event.PERSIST_STATE]
     data: Annotated[EventPersistStateData, Field(default_factory=lambda: EventPersistStateData(is_migrating=False))]
 
 
+@docs_group('Data structures')
 class SystemInfoEventData(BaseModel):
     mem_avg_bytes: Annotated[float, Field(alias='memAvgBytes')]
     mem_current_bytes: Annotated[float, Field(alias='memCurrentBytes')]
@@ -61,26 +64,31 @@ def to_crawlee_format(self) -> EventSystemInfoData:
         )
 
 
+@docs_group('Data structures')
 class SystemInfoEvent(BaseModel):
     name: Literal[Event.SYSTEM_INFO]
     data: SystemInfoEventData
 
 
+@docs_group('Data structures')
 class MigratingEvent(BaseModel):
     name: Literal[Event.MIGRATING]
     data: Annotated[EventMigratingData, Field(default_factory=EventMigratingData)]
 
 
+@docs_group('Data structures')
 class AbortingEvent(BaseModel):
     name: Literal[Event.ABORTING]
     data: Annotated[EventAbortingData, Field(default_factory=EventAbortingData)]
 
 
+@docs_group('Data structures')
 class ExitEvent(BaseModel):
     name: Literal[Event.EXIT]
     data: Annotated[EventExitData, Field(default_factory=EventExitData)]
 
 
+@docs_group('Data structures')
 class EventWithoutData(BaseModel):
     name: Literal[
         Event.SESSION_RETIRED,
@@ -93,11 +101,13 @@ class EventWithoutData(BaseModel):
     data: Any = None
 
 
+@docs_group('Data structures')
 class DeprecatedEvent(BaseModel):
     name: Literal['cpuInfo']
     data: Annotated[dict[str, Any], Field(default_factory=dict)]
 
 
+@docs_group('Data structures')
 class UnknownEvent(BaseModel):
     name: str
     data: Annotated[dict[str, Any], Field(default_factory=dict)]
@@ -125,6 +135,7 @@ class UnknownEvent(BaseModel):
 )
 
 
+@docs_group('Classes')
 class PlatformEventManager(EventManager):
     """A class for managing Actor events.
 

src/apify/_proxy_configuration.py

@@ -16,6 +16,7 @@
 from crawlee.proxy_configuration import _NewUrlFunction
 
 from apify._configuration import Configuration
+from apify._utils import docs_group
 from apify.log import logger
 
 if TYPE_CHECKING:
@@ -68,6 +69,7 @@ def _check(
         raise ValueError(f'{error_str} does not match pattern {pattern.pattern!r}')
 
 
+@docs_group('Classes')
 @dataclass
 class ProxyInfo(CrawleeProxyInfo):
     """Provides information about a proxy connection that is used for requests."""
@@ -87,6 +89,7 @@ class ProxyInfo(CrawleeProxyInfo):
     """
 
 
+@docs_group('Classes')
 class ProxyConfiguration(CrawleeProxyConfiguration):
     """Configures a connection to a proxy server with the provided options.
 

src/apify/_utils.py

@@ -3,6 +3,7 @@
 import builtins
 import sys
 from importlib import metadata
+from typing import Callable, Literal
 
 
 def get_system_info() -> dict:
@@ -24,3 +25,34 @@ def get_system_info() -> dict:
 
 def is_running_in_ipython() -> bool:
     return getattr(builtins, '__IPYTHON__', False)
+
+
+GroupName = Literal['Classes', 'Abstract classes', 'Data structures', 'Errors', 'Functions']
+
+
+def docs_group(group_name: GroupName) -> Callable:  # noqa: ARG001
+    """Decorator to mark symbols for rendering and grouping in documentation.
+
+    This decorator is used purely for documentation purposes and does not alter the behavior
+    of the decorated callable.
+    """
+
+    def wrapper(func: Callable) -> Callable:
+        return func
+
+    return wrapper
+
+
+def docs_name(symbol_name: str) -> Callable:  # noqa: ARG001
+    """Decorator for renaming symbols in documentation.
+
+    This changes the rendered name of the symbol only in the rendered web documentation.
+
+    This decorator is used purely for documentation purposes and does not alter the behavior
+    of the decorated callable.
+    """
+
+    def wrapper(func: Callable) -> Callable:
+        return func
+
+    return wrapper

src/apify/apify_storage_client/_apify_storage_client.py

@@ -5,6 +5,7 @@
 from crawlee.base_storage_client import BaseStorageClient
 
 from apify._configuration import Configuration
+from apify._utils import docs_group
 from apify.apify_storage_client._dataset_client import DatasetClient
 from apify.apify_storage_client._dataset_collection_client import DatasetCollectionClient
 from apify.apify_storage_client._key_value_store_client import KeyValueStoreClient
@@ -13,6 +14,7 @@
 from apify.apify_storage_client._request_queue_collection_client import RequestQueueCollectionClient
 
 
+@docs_group('Classes')
 class ApifyStorageClient(BaseStorageClient):
     """A storage client implementation based on the Apify platform storage."""
 

website/build_api_reference.sh

@@ -1,39 +1,4 @@
 #!/bin/bash
 
-# On macOS, sed requires a space between -i and '' to specify no backup should be done
-# On Linux, sed requires no space between -i and '' to specify no backup should be done
-sed_no_backup() {
-    if [[ $(uname) = "Darwin" ]]; then
-        sed -i '' "$@"
-    else
-        sed -i'' "$@"
-    fi
-}
-
-# Create docspec dump of this package's source code through pydoc-markdown
-python ./pydoc-markdown/generate_ast.py > docspec-dump.jsonl
-sed_no_backup "s#${PWD}/..#REPO_ROOT_PLACEHOLDER#g" docspec-dump.jsonl
-
-# Create docpec dump from the right version of the apify-shared package
-apify_shared_version=$(python -c "import apify_shared; print(apify_shared.__version__)")
-apify_shared_tempdir=$(realpath "$(mktemp -d)")
-git clone --quiet https://github.com/apify/apify-shared-python.git "${apify_shared_tempdir}"
-cp ./pydoc-markdown.yml "${apify_shared_tempdir}/pydoc-markdown.yml"
-sed_no_backup "s#search_path: \[../src\]#search_path: \[./src\]#g" "${apify_shared_tempdir}/pydoc-markdown.yml"
-
-(
-    cd "${apify_shared_tempdir}";
-    git checkout --quiet "v${apify_shared_version}";
-    pydoc-markdown --quiet --dump > ./apify-shared-docspec-dump.jsonl
-)
-
-cp "${apify_shared_tempdir}/apify-shared-docspec-dump.jsonl" .
-sed_no_backup "s#${apify_shared_tempdir}#REPO_ROOT_PLACEHOLDER#g" apify-shared-docspec-dump.jsonl
-
-rm -rf "${apify_shared_tempdir}"
-
 # Generate import shortcuts from the modules
 python generate_module_shortcuts.py
-
-# Transform the docpec dumps into Typedoc-compatible docs tree
-node transformDocs.js

website/docusaurus.config.js

@@ -1,7 +1,20 @@
-/* eslint-disable global-require,import/no-extraneous-dependencies */
+const path = require('path');
+
 const { config } = require('@apify/docs-theme');
+
 const { externalLinkProcessor } = require('./tools/utils/externalLink');
-const { groupSort } = require('./transformDocs.js');
+
+const GROUP_ORDER = [
+    'Classes',
+    'Data structures',
+];
+
+const groupSort = (g1, g2) => {
+    if (GROUP_ORDER.includes(g1) && GROUP_ORDER.includes(g2)) {
+        return GROUP_ORDER.indexOf(g1) - GROUP_ORDER.indexOf(g2);
+    }
+    return g1.localeCompare(g2);
+};
 
 const { absoluteUrl } = config;
 
@@ -15,6 +28,7 @@ module.exports = {
     projectName: 'apify-sdk-python',
     scripts: ['/js/custom.js'],
     favicon: 'img/favicon.ico',
+    githubHost: 'github.com',
     onBrokenLinks:
     /** @type {import('@docusaurus/types').ReportingSeverity} */ ('warn'),
     onBrokenMarkdownLinks:
@@ -83,10 +97,27 @@ module.exports = {
                 typedocOptions: {
                     excludeExternals: false,
                 },
-                pathToCurrentVersionTypedocJSON: `${__dirname}/api-typedoc-generated.json`,
                 sortSidebar: groupSort,
                 routeBasePath: 'reference',
                 python: true,
+                pythonOptions: {
+                    pythonModulePath: path.join(__dirname, '../src/apify'),
+                    moduleShortcutsPath: path.join(__dirname, '/module_shortcuts.json'),
+                },
+                reexports: [
+                    {
+                        url: 'https://crawlee.dev/python/api/class/Dataset',
+                        group: 'Classes',
+                    },
+                    {
+                        url: 'https://crawlee.dev/python/api/class/KeyValueStore',
+                        group: 'Classes',
+                    },
+                    {
+                        url: 'https://crawlee.dev/python/api/class/RequestQueue',
+                        group: 'Classes',
+                    },
+                ],
             },
         ],
         ...config.plugins,