8000 ci: run the checks through Nx, and only for the projects a change affects by adeelahmad · Pull Request #303 · adeelahmad/lense · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,14 @@ updates:
# presets, so it only builds inside AFFiNE's own repo. The notes editor stays on 0.19.5 until that changes.
- dependency-name: "@blocksuite/*"

- package-ecosystem: npm # pnpm: Nx, the root's only dependency
directory: /
schedule:
interval: monthly
commit-message:
prefix: build
include: scope

- package-ecosystem: github-actions
directory: /
schedule:
Expand Down
160 changes: 142 additions & 18 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ on:
push:
branches: [main]
pull_request:
# by hand: every project, not only the affected ones
workflow_dispatch:

# Pushes to a pull request cancel its older runs
concurrency:
Expand All @@ -12,12 +14,70 @@ concurrency:

permissions:
contents: read
actions: read # to find main's last green run, the base a push to main is compared with

# Nx (nx.json, and each project's project.json) runs the checks. The first job asks it which projects the change
# affects; each job below runs only when its project is one of them, and a job that doesn't run counts as passed.
# A pull request is compared with the main it merges into, a push to main with main's last green CI run.
#
# Each task runs fresh here: Nx's cache is for a contributor's machine, and CI neither reads nor fills one.
env:
NX_SKIP_NX_CACHE: "true"

# Every workflow runs on GitHub's ubuntu-latest. To use a self-hosted runner instead, set the repository variable
# CI_RUNNER (Settings → Secrets and variables → Actions → Variables) to its labels as JSON: ["self-hosted","Linux","X64"]
jobs:
affected:
name: Affected projects
runs-on: ${{ fromJSON(vars.CI_RUNNER || '"ubuntu-latest"') }}
outputs:
projects: ${{ steps.affected.outputs.projects }}
others: ${{ steps.affected.outputs.others }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0 # the history back to the base the change is compared with
filter: tree:0
- uses: pnpm/action-setup@v6
with:
run_install: false
- uses: actions/setup-node@v7
with:
node-version: "22"
cache: ${{ runner.environment == 'github-hosted' && 'pnpm' || '' }}
cache-dependency-path: pnpm-lock.yaml
- name: Install Nx
run: pnpm install --frozen-lockfile
- name: The commits to compare
if: github.event_name != 'workflow_dispatch'
uses: nrwl/nx-set-shas@v5
- name: The projects the change affects
id: affected
run: |
# anything but two JSON lists fails this job, and then every job runs
set -o pipefail
if [ "$GITHUB_EVENT_NAME" = workflow_dispatch ]; then
projects=$(pnpm exec nx show projects --json | jq -c .)
others=$(pnpm exec nx show projects --exclude=backend,frontend --json | jq -c .)
else
projects=$(pnpm exec nx show projects --affected --json | jq -c .)
others=$(pnpm exec nx show projects --affected --exclude=backend,frontend --json | jq -c .)
fi
jq -en --argjson p "$projects" --argjson o "$others" '($p | type) == "array" and ($o | type) == "array"' >/dev/null
echo "projects=$projects" >> "$GITHUB_OUTPUT"
echo "others=$others" >> "$GITHUB_OUTPUT"
{
echo "### Affected projects"
if [ -n "${NX_BASE:-}" ]; then echo "Comparing \`${NX_BASE:0:12}\` with \`${NX_HEAD:0:12}\`."; else echo "Run by hand: every project."; fi
echo
echo "$projects" | jq -r 'if length == 0 then "None: nothing to build or test." else map("- " + .) | join("\n") end'
} >> "$GITHUB_STEP_SUMMARY"

backend:
name: FastAPI (embedded SurrealDB)
needs: affected
# when the affected job itself fails, run anyway
if: ${{ !cancelled() && (needs.affected.result != 'success' || contains(needs.affected.outputs.projects, '"backend"')) }}
runs-on: ${{ fromJSON(vars.CI_RUNNER || '"ubuntu-latest"') }}
defaults:
run:
Expand Down Expand Up @@ -67,26 +127,37 @@ jobs:
cache-local-path: ${{ runner.environment == 'self-hosted' && format('{0}/uv-cache', runner.tool_cache) || '' }}
python-version: "3.12"

- uses: pnpm/action-setup@v6
with:
run_install: false
- uses: actions/setup-node@v7
with:
node-version: "22"
cache: ${{ runner.environment == 'github-hosted' && 'pnpm' || '' }}
cache-dependency-path: pnpm-lock.yaml
- name: Install Nx
working-directory: .
run: pnpm install --frozen-lockfile

- name: Install
run: uv sync --frozen

# the commands are the targets in fastapi_backend/project.json
- name: Lint
run: uv run ruff check . && uv run ruff format --check .
run: pnpm exec nx run backend:lint

- name: Type check
run: uv run mypy
run: pnpm exec nx run backend:typecheck

- name: OpenAPI schema is up to date
run: |
OPENAPI_OUTPUT_FILE=/tmp/openapi.json uv run python -m commands.generate_openapi_schema
diff -u ../nextjs-frontend/openapi.json /tmp/openapi.json
run: pnpm exec nx run backend:openapi-check

- name: Tests
# the suite took 28 min on the self-hosted runner at 544 tests (3 min on a laptop); it keeps growing
timeout-minutes: 60
run: |
set -o pipefail
uv run pytest -n auto -p no:cacheprovider --durations=20 --cov=app --cov-report=xml:coverage.xml --cov-report=term | tee pytest.txt
pnpm exec nx run backend:test:ci | tee pytest.txt
{ echo "### Backend coverage"; echo '```'; grep -E '^(Name|TOTAL)' pytest.txt; echo '```'; } >> "$GITHUB_STEP_SUMMARY"

- uses: actions/upload-artifact@v7
Expand All @@ -96,6 +167,8 @@ jobs:

backend-surrealdb-server:
name: FastAPI (SurrealDB server)
needs: affected
if: ${{ !cancelled() && (needs.affected.result != 'success' || contains(needs.affected.outputs.projects, '"backend"')) }}
runs-on: ${{ fromJSON(vars.CI_RUNNER || '"ubuntu-latest"') }}
defaults:
run:
Expand Down Expand Up @@ -152,48 +225,99 @@ jobs:
# a cache that outlives the job on the self-hosted runner (the default is in the job's temp folder)
cache-local-path: ${{ runner.environment == 'self-hosted' && format('{0}/uv-cache', runner.tool_cache) || '' }}
python-version: "3.12"
- uses: pnpm/action-setup@v6
with:
run_install: false
- uses: actions/setup-node@v7
with:
node-version: "22"
cache: ${{ runner.environment == 'github-hosted' && 'pnpm' || '' }}
cache-dependency-path: pnpm-lock.yaml
- name: Install Nx
working-directory: .
run: pnpm install --frozen-lockfile
- run: uv sync --frozen
- name: Tests against the server
# slower than the embedded run: 30 min covered two thirds of it on the self-hosted runner
timeout-minutes: 75
run: uv run pytest -n 4 --durations=20
run: pnpm exec nx run backend:test:server
- name: Remove SurrealDB
if: always()
run: docker rm -f "$SURREAL_CONTAINER" || true

frontend:
name: Next.js
needs: affected
if: ${{ !cancelled() && (needs.affected.result != 'success' || contains(needs.affected.outputs.projects, '"frontend"')) }}
runs-on: ${{ fromJSON(vars.CI_RUNNER || '"ubuntu-latest"') }}
defaults:
run:
working-directory: ./nextjs-frontend
steps:
- uses: actions/checkout@v4
- uses: pnpm/action-setup@v6
with:
# the pnpm version comes from packageManager in the frontend's package.json
package_json_file: nextjs-frontend/package.json
# the pnpm version comes from packageManager in package.json (the same in the frontend's)
run_install: false
- uses: actions/setup-node@v7
with:
node-version: "22"
# on the self-hosted runner pnpm's store stays on disk; saving it to GitHub's cache took minutes
cache: ${{ runner.environment == 'github-hosted' && 'pnpm' || '' }}
cache-dependency-path: nextjs-frontend/pnpm-lock.yaml
- run: pnpm install --frozen-lockfile
- run: pnpm lint
- run: pnpm tsc
cache-dependency-path: |
pnpm-lock.yaml
nextjs-frontend/pnpm-lock.yaml
- name: Install Nx
run: pnpm install --frozen-lockfile
- name: Install
working-directory: ./nextjs-frontend
run: pnpm install --frozen-lockfile
# the commands are the targets in nextjs-frontend/project.json
- name: Lint
run: pnpm exec nx run frontend:lint
- name: Type check
run: pnpm exec nx run frontend:typecheck
- name: Tests
run: |
set -o pipefail
pnpm coverage --coverageReporters=text-summary --coverageReporters=lcov | tee coverage.txt
pnpm exec nx run frontend:test:ci | tee coverage.txt
{ echo "### Frontend coverage"; echo '```'; sed -n '/Coverage summary/,/^=*$/p' coverage.txt; echo '```'; } >> "$GITHUB_STEP_SUMMARY"
- uses: actions/upload-artifact@v7
with:
name: frontend-coverage
path: nextjs-frontend/coverage/lcov.info
- name: Build
run: pnpm build
run: pnpm exec nx run frontend:build
env:
API_BASE_URL: http://localhost:8000
AUTH_SECRET: ci-auth-secret

projects:
name: Other projects
needs: affected
# every other project with lint, typecheck, test or build targets: a new project.json is all it takes to join
if: ${{ !cancelled() && (needs.affected.result != 'success' || needs.affected.outputs.others != '[]') }}
runs-on: ${{ fromJSON(vars.CI_RUNNER || '"ubuntu-latest"') }}
env:
OTHERS: ${{ needs.affected.outputs.others }}
steps:
- uses: actions/checkout@v4
- uses: pnpm/action-setup@v6
with:
run_install: false
- uses: actions/setup-node@v7
with:
node-version: "22"
cache: ${{ runner.environment == 'github-hosted' && 'pnpm' || '' }}
cache-dependency-path: pnpm-lock.yaml
- name: Install Nx
run: pnpm install --frozen-lockfile
- uses: astral-sh/setup-uv@v7
with:
cache-local-path: ${{ runner.environment == 'self-hosted' && format('{0}/uv-cache', runner.tool_cache) || '' }}
python-version: "3.12"
- name: Lint, type check, test and build them
run: |
# all of them when the affected job didn't say
if [ -n "$OTHERS" ]; then
pnpm exec nx run-many -t lint typecheck test build --outputStyle=static -p "$(echo "$OTHERS" | jq -r 'join(",")')"
else
pnpm exec nx run-many -t lint typecheck test build --outputStyle=static --exclude=backend,frontend
fi
7 changes: 3 additions & 4 deletions .github/workflows/docs.yml
Original file line number Diff line number Diff line change
@@ -1,13 +1,12 @@
name: docs

# Builds the documentation site on every change to docs, and publishes it to GitHub Pages once Pages is turned on
# (Settings → Pages → Source: GitHub Actions). Until then it only builds.
# Builds the documentation site on every change to docs on main, and publishes it to GitHub Pages once Pages is turned
# on (Settings → Pages → Source: GitHub Actions). Until then it only builds. On a pull request the site is built by CI
# (ci.yml, Other projects: the docs project's build target in docs/project.json), when the change affects it.
on:
push:
branches: [main]
paths: [docs/**, mkdocs.yml, overrides/**, README.md, CHANGELOG.md, .github/workflows/docs.yml]
pull_request:
paths: [docs/**, mkdocs.yml, overrides/**, README.md, CHANGELOG.md, .github/workflows/docs.yml]
workflow_dispatch:

concurrency:
Expand Down
4 changes: 4 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -173,3 +173,7 @@ local-shared-data/*
.lens-build
.lens-sandbox
.lens-control/

# Nx (package.json, nx.json): the root's node_modules is Nx itself; .nx holds its local cache and project graph
/node_modules/
/.nx/
3 changes: 3 additions & 0 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,9 @@ review, and charts.
- **Fix every red check** in the PR, flaky or unrelated ones included. Never skip or disable a test, and never just
re-run and wait.
- **Checks run in CI.** Don't ask the owner to run things locally.
- **Nx runs the checks.** `make affected` runs the lint, type check, test and build targets of every project a change
touches, as CI does. A new project joins CI with a `project.json`
([Monorepo and CI](docs/contributing.md#monorepo-and-ci-nx)).
- **Generated files come from tooling** (`make openapi`), never from hand-merging.
- **Docs move with the code.** Update `docs/` when behaviour or configuration changes; `mkdocs build --strict` stays
clean.
Expand Down
3 changes: 2 additions & 1 deletion CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ Thanks for helping. The full guide, with the backend and frontend conventions, i
## Make the change

- Backend: `make test-backend` and `make lint-backend` (ruff, mypy). Frontend: `make test-frontend` and
`make lint-frontend`.
`make lint-frontend`. `make affected` checks every project your change touches. [Nx](https://nx.dev/) runs them,
the same way CI does; see [Monorepo and CI](docs/contributing.md#monorepo-and-ci-nx).
- Add tests with the change, and update `docs/` when behaviour or configuration changes.
- Changing the shape of stored data? Existing installs have real data: add an upgrade step
(docs/database.md, Upgrades) rather than expecting a fresh database.
Expand Down
44 changes: 27 additions & 17 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,16 @@ DOCKER_COMPOSE ?= $(shell docker compose version >/dev/null 2>&1 && echo docker
help: ## List the commands
@awk '/^[a-zA-Z_-]+:.*##/{split($$1, target, ":"); print " " target[1] "\t" substr($$0, index($$0,"##")+3)}' $(MAKEFILE_LIST)

# Local development
.PHONY: start-backend test-backend lint-backend start-frontend test-frontend lint-frontend openapi worker docs docs-build hooks
# Local development. The checks run through Nx (nx.json and each project's project.json), as CI runs them, and come
# back from its local cache (.nx/cache) when nothing they read has changed. Nx itself is the root package.json's one
# dependency, installed on first use.
NX = pnpm exec nx

.PHONY: start-backend test-backend lint-backend start-frontend test-frontend lint-frontend openapi worker docs docs-build hooks affected

node_modules/.bin/nx: package.json pnpm-lock.yaml
pnpm install --frozen-lockfile
@touch $@

hooks: ## Install the git hooks: lint on commit, Conventional Commit messages
cd $(BACKEND_DIR) && uv run pre-commit install -c ../.pre-commit-config.yaml
Expand All @@ -22,30 +30,32 @@ start-backend: $(BACKEND_DIR)/.env $(BACKEND_DIR)/archive.yaml ## Start the API
worker: ## Run a job worker against the configured database
cd $(BACKEND_DIR) && uv run lens worker

test-backend: ## Run the backend tests
cd $(BACKEND_DIR) && uv run pytest -n auto
test-backend: node_modules/.bin/nx ## Run the backend tests
$(NX) run backend:test

lint-backend: ## Lint, format check and type check the backend
cd $(BACKEND_DIR) && uv run ruff check . && uv run ruff format --check . && uv run mypy
lint-backend: node_modules/.bin/nx ## Lint, format check and type check the backend
$(NX) run-many -t lint typecheck -p backend

start-frontend: $(FRONTEND_DIR)/.env.local ## Start the web app with hot reload
cd $(FRONTEND_DIR) && ./start.sh

test-frontend: ## Run the frontend tests
cd $(FRONTEND_DIR) && pnpm run test
test-frontend: node_modules/.bin/nx ## Run the frontend tests
$(NX) run frontend:test

lint-frontend: node_modules/.bin/nx ## Lint and type check the frontend
$(NX) run-many -t lint typecheck -p frontend

lint-frontend: ## Lint and type check the frontend
cd $(FRONTEND_DIR) && pnpm run lint && pnpm run tsc
openapi: node_modules/.bin/nx ## Regenerate the OpenAPI schema and the frontend client
$(NX) run backend:openapi

openapi: ## Regenerate the OpenAPI schema and the frontend client
cd $(BACKEND_DIR) && uv run python -m commands.generate_openapi_schema
cd $(FRONTEND_DIR) && pnpm run generate-client
docs: node_modules/.bin/nx ## Serve the documentation (docs-build checks it the way CI does)
$(NX) run docs:serve

docs: ## Serve the documentation (docs-build checks it the way CI does)
cd $(BACKEND_DIR) && uv run mkdocs serve -f ../mkdocs.yml
docs-build: node_modules/.bin/nx ## Build the documentation site, failing on broken links
$(NX) run docs:build

docs-build: ## Build the documentation site, failing on broken links
cd $(BACKEND_DIR) && uv run mkdocs build --strict -f ../mkdocs.yml --site-dir ../site
affected: node_modules/.bin/nx ## Lint, type check, test and build only the projects your changes affect, against main
$(NX) affected -t lint typecheck test build

# Run it (Docker)
PROD_COMPOSE=$(DOCKER_COMPOSE) -f docker-compose.prod.yml
Expand Down
Loading
Loading
0