diff --git a/.github/workflows/_check_docstrings.yaml b/.github/workflows/_check_docstrings.yaml index ba24c63d..7118cf3b 100644 --- a/.github/workflows/_check_docstrings.yaml +++ b/.github/workflows/_check_docstrings.yaml @@ -30,7 +30,7 @@ jobs: python-version: ${{ env.PYTHON_VERSION }} - name: Install dependencies - run: make install-dev + run: uv run poe install-dev - name: Async docstrings check - run: make check-async-docstrings + run: uv run poe check-async-docstrings diff --git a/.github/workflows/_release_docs.yaml b/.github/workflows/_release_docs.yaml index af809838..afd9abea 100644 --- a/.github/workflows/_release_docs.yaml +++ b/.github/workflows/_release_docs.yaml @@ -68,13 +68,10 @@ jobs: python-version: ${{ env.PYTHON_VERSION }} - name: Install Python dependencies - run: make install-dev + run: uv run poe install-dev - - name: Build generated API reference - run: make build-api-reference - - - name: Build Docusaurus docs - run: make build-docs + - name: Build docs + run: uv run poe build-docs env: APIFY_SIGNING_TOKEN: ${{ secrets.APIFY_SIGNING_TOKEN }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e909d818..497f4d1c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,18 +3,18 @@ repos: hooks: - id: lint-check name: Lint check - entry: make lint + entry: uv run poe lint language: system pass_filenames: false - id: type-check name: Type check - entry: make type-check + entry: uv run poe type-check language: system pass_filenames: false - id: docstrings-check name: Docstrings check - entry: make check-async-docstrings + entry: uv run poe check-async-docstrings language: system pass_filenames: false diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0c4cd173..9f2437e8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,12 +8,35 @@ For local development, it is required to have Python 3.10 (or a later version) i We use [uv](https://docs.astral.sh/uv/) for project management. Install it and set up your IDE accordingly. +We use [Poe the Poet](https://poethepoet.natn.io/) as a task runner, similar to npm scripts in `package.json`. +All tasks are defined in `pyproject.toml` under `[tool.poe.tasks]` and can be run with `uv run poe `. + +### Available tasks + +| Task | Description | +| ---- | ----------- | +| `install-dev` | Install development dependencies | +| `check-code` | Run lint, type-check, unit-tests, and docstring check | +| `lint` | Run linter | +| `format` | Fix lint issues and format code | +| `type-check` | Run type checker | +| `unit-tests` | Run unit tests | +| `unit-tests-cov` | Run unit tests with coverage | +| `integration-tests` | Run integration tests | +| `integration-tests-cov` | Run integration tests with coverage | +| `check-async-docstrings` | Check async client docstrings | +| `fix-async-docstrings` | Fix async client docstrings | +| `build-docs` | Build documentation website | +| `run-docs` | Run documentation website locally | +| `build` | Build package | +| `clean` | Remove build artifacts and clean caches | + ## Dependencies To install this package and its development dependencies, run: ```sh -make install-dev +uv run poe install-dev ``` ## Code checking @@ -21,7 +44,7 @@ make install-dev To execute all code checking tools together, run: ```sh -make check-code +uv run poe check-code ``` ### Linting @@ -31,7 +54,7 @@ We utilize [ruff](https://docs.astral.sh/ruff/) for linting, which analyzes code To run linting: ```sh -make lint +uv run poe lint ``` ### Formatting @@ -41,7 +64,7 @@ Our automated code formatting also leverages [ruff](https://docs.astral.sh/ruff/ To run formatting: ```sh -make format +uv run poe format ``` ### Type checking @@ -51,81 +74,51 @@ Type checking is handled by [ty](https://docs.astral.sh/ty/), verifying code aga To run type checking: ```sh -make type-check +uv run poe type-check ``` ### Unit tests -We employ pytest as our testing framework, equipped with various plugins. Check pyproject.toml for configuration details and installed plugins. - We use [pytest](https://docs.pytest.org/) as a testing framework with many plugins. Check `pyproject.toml` for configuration details and installed plugins. To run unit tests: ```sh -make unit-tests +uv run poe unit-tests ``` -To run unit tests with HTML coverage report: +To run unit tests with XML coverage report: ```sh -make unit-tests-cov +uv run poe unit-tests-cov ``` ## Integration tests -We have integration tests which build and run Actors using the Python SDK on the Apify Platform. To run these tests, -you need to set the `APIFY_TEST_USER_API_TOKEN` environment variable to the API token of the Apify user you want to -use for the tests, and then start them with `make integration-tests`. - -For subset of integration tests another token is needed `APIFY_TEST_USER_2_API_TOKEN`. Such tests are testing -the storage restricted access and thus need two user accounts. - -If you want to run the integration tests on a different environment than the main Apify Platform, you need to set -the `APIFY_INTEGRATION_TESTS_API_URL` environment variable to the right URL to the Apify API you want to use. - -## Documentation - -We adhere to the [Google docstring format](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) for documenting our codebase. Every user-facing class or method is documented. Documentation standards are enforced using [Ruff](https://docs.astral.sh/ruff/). +We have integration tests which interact with the Apify platform. -Our API documentation is generated from these docstrings using [pydoc-markdown](https://pypi.org/project/pydoc-markdown/) with additional post-processing. Markdown files in the `docs/` folder complement the autogenerated content. Final documentation is rendered using [Docusaurus](https://docusaurus.io/) and published to GitHub Pages. +Prerequisites: -To run the documentation locally, you need to have Node.js version 20 or higher installed. Once you have the correct version of Node.js, follow these steps: +- Set `APIFY_TEST_USER_API_TOKEN` to your Apify API token +- Set `APIFY_TEST_USER_2_API_TOKEN` for storage access tests (requires two accounts) +- Optionally set `APIFY_INTEGRATION_TESTS_API_URL` to use a different Apify API environment -Navigate to the `website/` directory: +To run integration tests: ```sh -cd website/ +uv run poe integration-tests ``` -Enable Corepack, which installs Yarn automatically: - -```sh -corepack enable -``` - -Build the API reference: - -```sh -./build_api_reference.sh -``` - -Install the necessary dependencies: - -```sh -yarn -``` +## Documentation -Start the project in development mode with Hot Module Replacement (HMR): +We adhere to the [Google docstring format](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) for documenting our codebase. Every user-facing class or method is documented. Documentation standards are enforced using [Ruff](https://docs.astral.sh/ruff/). -```sh -yarn start -``` +Our API documentation is generated from these docstrings using [pydoc-markdown](https://pypi.org/project/pydoc-markdown/) with additional post-processing. Markdown files in the `docs/` folder complement the autogenerated content. Final documentation is rendered using [Docusaurus](https://docusaurus.io/) and published to GitHub Pages. -Or using `make`: +To run the documentation locally (requires Node.js): ```sh -make run-doc +uv run poe run-docs ``` ## Release process @@ -150,14 +143,14 @@ name = "apify_client" version = "x.z.y" ``` -4. Generate the distribution archives for the package: +4. Build the package: -```shell -uv build +```sh +uv run poe build ``` -5. Set up the PyPI API token for authentication and upload the package to PyPI: +5. Upload to PyPI: -```shell +```sh uv publish --token YOUR_API_TOKEN ``` diff --git a/Makefile b/Makefile deleted file mode 100644 index ea928ca8..00000000 --- a/Makefile +++ /dev/null @@ -1,78 +0,0 @@ -.PHONY: clean install-dev build publish-to-pypi lint type-check unit-tests unit-tests-cov integration-tests \ - integration-tests-cov format check-async-docstrings check-code fix-async-docstrings build-api-reference \ - build-docs run-docs - -# This is default for local testing, but GitHub workflows override it to a higher value in CI -INTEGRATION_TESTS_CONCURRENCY = 1 - -clean: - rm -rf .ty_cache .pytest_cache .ruff_cache build dist htmlcov .coverage - -install-dev: - uv sync --all-extras - uv run pre-commit install - -build: - uv build --verbose - -# APIFY_PYPI_TOKEN_CRAWLEE is expected to be set in the environment -publish-to-pypi: - uv publish --verbose --token "${APIFY_PYPI_TOKEN_CRAWLEE}" - -lint: - uv run ruff format --check - uv run ruff check - -type-check: - uv run ty check - -unit-tests: - uv run pytest \ - --numprocesses=auto \ - --verbose \ - tests/unit - -unit-tests-cov: - uv run pytest \ - --numprocesses=auto \ - --verbose \ - --cov=src/apify_client \ - --cov-report=xml:coverage-unit.xml \ - tests/unit - -integration-tests: - uv run pytest \ - --numprocesses=$(INTEGRATION_TESTS_CONCURRENCY) \ - --verbose \ - tests/integration - -integration-tests-cov: - uv run pytest \ - --numprocesses=$(INTEGRATION_TESTS_CONCURRENCY) \ - --verbose \ - --cov=src/apify_client \ - --cov-report=xml:coverage-integration.xml \ - tests/integration - -format: - uv run ruff check --fix - uv run ruff format - -check-async-docstrings: - uv run python scripts/check_async_docstrings.py - -# The check-code target runs a series of checks equivalent to those performed by pre-commit hooks -# and the run_checks.yaml GitHub Actions workflow. -check-code: lint type-check unit-tests check-async-docstrings - -fix-async-docstrings: - uv run python scripts/fix_async_docstrings.py - -build-api-reference: - cd website && uv run ./build_api_reference.sh - -build-docs: - cd website && uv run npm clean-install && uv run npm run build - -run-docs: build-api-reference - cd website && uv run npm clean-install && uv run npm run start diff --git a/pyproject.toml b/pyproject.toml index 7d135dd6..f9e31a88 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,7 @@ dependencies = [ dev = [ "dycw-pytest-only<3.0.0", "griffe", + "poethepoet<1.0.0", "pre-commit<5.0.0", "pydoc-markdown<5.0.0", "pytest-asyncio<2.0.0", @@ -178,3 +179,35 @@ exclude_lines = ["pragma: no cover", "if TYPE_CHECKING:", "assert_never()"] [tool.ipdb] context = 7 + +# Run tasks with: uv run poe +[tool.poe.tasks] +clean = "rm -rf .coverage .pytest_cache .ruff_cache .ty_cache build dist htmlcov" +install-sync = "uv sync --all-extras" +build = "uv build --verbose" +publish-to-pypi = "uv publish --verbose --token ${APIFY_PYPI_TOKEN_CRAWLEE}" +type-check = "uv run ty check" +unit-tests = "uv run pytest --numprocesses=auto --verbose tests/unit" +unit-tests-cov = "uv run pytest --numprocesses=auto --verbose --cov=src/apify_client --cov-report=xml:coverage-unit.xml tests/unit" +integration-tests = "uv run pytest --numprocesses=${INTEGRATION_TESTS_CONCURRENCY:-1} --verbose tests/integration" +integration-tests-cov = "uv run pytest --numprocesses=${INTEGRATION_TESTS_CONCURRENCY:-1} --verbose --cov=src/apify_client --cov-report=xml:coverage-integration.xml tests/integration" +check-async-docstrings = "uv run python scripts/check_async_docstrings.py" +fix-async-docstrings = "uv run python scripts/fix_async_docstrings.py" +check-code = ["lint", "type-check", "unit-tests", "check-async-docstrings"] + +[tool.poe.tasks.install-dev] +shell = "uv sync --all-extras && uv run pre-commit install" + +[tool.poe.tasks.lint] +shell = "uv run ruff format --check && uv run ruff check" + +[tool.poe.tasks.format] +shell = "uv run ruff check --fix && uv run ruff format" + +[tool.poe.tasks.build-docs] +shell = "./build_api_reference.sh && npm ci && npm run build" +cwd = "website" + +[tool.poe.tasks.run-docs] +shell = "./build_api_reference.sh && npm ci && npm run start" +cwd = "website" diff --git a/scripts/check_async_docstrings.py b/scripts/check_async_docstrings.py index 95323e4a..311a6a46 100755 --- a/scripts/check_async_docstrings.py +++ b/scripts/check_async_docstrings.py @@ -54,7 +54,9 @@ if found_issues: print() - print('Issues with async docstrings found. Please fix them manually or by running `make fix-async-docstrings`.') + print( + 'Issues with async docstrings found. Please fix them manually or by running `uv run poe fix-async-docstrings`.' + ) sys.exit(1) else: print('Success: async method docstrings are in sync with sync method docstrings.') diff --git a/uv.lock b/uv.lock index 72d47733..f30afd80 100644 --- a/uv.lock +++ b/uv.lock @@ -17,6 +17,7 @@ dependencies = [ dev = [ { name = "dycw-pytest-only" }, { name = "griffe" }, + { name = "poethepoet" }, { name = "pre-commit" }, { name = "pydoc-markdown" }, { name = "pytest" }, @@ -45,6 +46,7 @@ requires-dist = [ dev = [ { name = "dycw-pytest-only", specifier = "<3.0.0" }, { name = "griffe" }, + { name = "poethepoet", specifier = "<1.0.0" }, { name = "pre-commit", specifier = "<5.0.0" }, { name = "pydoc-markdown", specifier = "<5.0.0" }, { name = "pytest", specifier = "<9.0.0" }, @@ -757,6 +759,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, ] +[[package]] +name = "pastel" +version = "0.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/76/f1/4594f5e0fcddb6953e5b8fe00da8c317b8b41b547e2b3ae2da7512943c62/pastel-0.2.1.tar.gz", hash = "sha256:e6581ac04e973cac858828c6202c1e1e81fee1dc7de7683f3e1ffe0bfd8a573d", size = 7555, upload-time = "2020-09-16T19:21:12.43Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/aa/18/a8444036c6dd65ba3624c63b734d3ba95ba63ace513078e1580590075d21/pastel-0.2.1-py2.py3-none-any.whl", hash = "sha256:4349225fcdf6c2bb34d483e523475de5bb04a5c10ef711263452cb37d7dd4364", size = 5955, upload-time = "2020-09-16T19:21:11.409Z" }, +] + [[package]] name = "pathspec" version = "1.0.3" @@ -784,6 +795,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "poethepoet" +version = "0.40.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pastel" }, + { name = "pyyaml" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/38/9d/054c8435b03324ed9abd5d5ab8c45065b1f42c23952cd23f13a5921d8465/poethepoet-0.40.0.tar.gz", hash = "sha256:91835f00d03d6c4f0e146f80fa510e298ad865e7edd27fe4cb9c94fdc090791b", size = 81114, upload-time = "2026-01-05T19:09:13.116Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/bc/73327d12b176abea7a3c6c7d760e1a953992f7b59d72c0354e39d7a353b5/poethepoet-0.40.0-py3-none-any.whl", hash = "sha256:afd276ae31d5c53573c0c14898118d4848ccee3709b6b0be6a1c6cbe522bbc8a", size = 106672, upload-time = "2026-01-05T19:09:11.536Z" }, +] + [[package]] name = "pre-commit" version = "4.5.1"