From ddcd229cbe763f35db9d58c7b391369db77f4aaa Mon Sep 17 00:00:00 2001 From: Esteban Maya Cadavid Date: Thu, 15 Feb 2024 17:21:22 -0500 Subject: [PATCH 1/6] :recycle: [EDIT]: Refactor items and services endpoints and added support to test in actions --- .github/workflows/test.yaml | 30 +++++++++++++++++++ .../app/app/api/api_v1/endpoints/items.py | 15 ++++++---- .../app/app/api/api_v1/endpoints/users.py | 12 ++++++-- src/backend/app/app/models.py | 11 +++++++ .../app/app/tests/api/api_v1/test_users.py | 5 ++-- src/docker-compose.override.yml | 20 ++++++++----- src/docker-compose.yml | 20 ++++++------- 7 files changed, 85 insertions(+), 28 deletions(-) create mode 100644 .github/workflows/test.yaml diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000000..38e822af5f --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,30 @@ +name: Pytest Backend + +on: + push: + branches: + - master + +jobs: + + pytest: + runs-on: ubuntu-latest + defaults: + run: + working-directory: src + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Setup python 3.10 + uses: actions/setup-python@v2 + with: + python-version: '3.10' + + - name: docker-compose setup + run: | + docker-compose up -d + + - name: docker-compose run pytest + run: | + docker exec src_backend_1 pytest --cov=app --cov-report=term-missing app/tests diff --git a/src/backend/app/app/api/api_v1/endpoints/items.py b/src/backend/app/app/api/api_v1/endpoints/items.py index e254cda368..34364b614b 100644 --- a/src/backend/app/app/api/api_v1/endpoints/items.py +++ b/src/backend/app/app/api/api_v1/endpoints/items.py @@ -1,15 +1,15 @@ from typing import Any from fastapi import APIRouter, HTTPException -from sqlmodel import select +from sqlmodel import select, func from app.api.deps import CurrentUser, SessionDep -from app.models import Item, ItemCreate, ItemOut, ItemUpdate, Message +from app.models import Item, ItemCreate, ItemOut, ItemUpdate, Message, ItemsOut router = APIRouter() -@router.get("/", response_model=list[ItemOut]) +@router.get("/", response_model=ItemsOut) def read_items( session: SessionDep, current_user: CurrentUser, skip: int = 0, limit: int = 100 ) -> Any: @@ -17,9 +17,12 @@ def read_items( Retrieve items. """ + statment = select(func.count()).select_from(Item) + count = session.exec(statment).one() + if current_user.is_superuser: statement = select(Item).offset(skip).limit(limit) - return session.exec(statement).all() + items = session.exec(statement).all() else: statement = ( select(Item) @@ -27,7 +30,9 @@ def read_items( .offset(skip) .limit(limit) ) - return session.exec(statement).all() + items = session.exec(statement).all() + + return ItemsOut(data=items, count=count) @router.get("/{id}", response_model=ItemOut) diff --git a/src/backend/app/app/api/api_v1/endpoints/users.py b/src/backend/app/app/api/api_v1/endpoints/users.py index e3ec0fd34f..8ea973e287 100644 --- a/src/backend/app/app/api/api_v1/endpoints/users.py +++ b/src/backend/app/app/api/api_v1/endpoints/users.py @@ -1,7 +1,7 @@ from typing import Any, List from fastapi import APIRouter, Depends, HTTPException -from sqlmodel import select +from sqlmodel import select, func from app import crud from app.api.deps import ( @@ -16,6 +16,7 @@ UserCreate, UserCreateOpen, UserOut, + UsersOut, UserUpdate, UserUpdateMe, ) @@ -27,15 +28,20 @@ @router.get( "/", dependencies=[Depends(get_current_active_superuser)], - response_model=List[UserOut], + response_model=UsersOut ) def read_users(session: SessionDep, skip: int = 0, limit: int = 100) -> Any: """ Retrieve users. """ + + statment = select(func.count()).select_from(User) + count = session.exec(statment).one() + statement = select(User).offset(skip).limit(limit) users = session.exec(statement).all() - return users + + return UsersOut(data=users, count=count) @router.post( diff --git a/src/backend/app/app/models.py b/src/backend/app/app/models.py index bb03308a70..33ba253e8a 100644 --- a/src/backend/app/app/models.py +++ b/src/backend/app/app/models.py @@ -46,6 +46,11 @@ class UserOut(UserBase): id: int +class UsersOut(BaseModel): + data: list[UserOut] + count: int + + # Shared properties class ItemBase(SQLModel): title: str @@ -75,6 +80,12 @@ class Item(ItemBase, table=True): # Properties to return via API, id is always required class ItemOut(ItemBase): id: int + owner_id: int + + +class ItemsOut(BaseModel): + data: list[ItemOut] + count: int # Generic message diff --git a/src/backend/app/app/tests/api/api_v1/test_users.py b/src/backend/app/app/tests/api/api_v1/test_users.py index ba22bfe969..44d97f6649 100644 --- a/src/backend/app/app/tests/api/api_v1/test_users.py +++ b/src/backend/app/app/tests/api/api_v1/test_users.py @@ -110,6 +110,7 @@ def test_retrieve_users( r = client.get(f"{settings.API_V1_STR}/users/", headers=superuser_token_headers) all_users = r.json() - assert len(all_users) > 1 - for item in all_users: + assert len(all_users["data"]) > 1 + assert "count" in all_users + for item in all_users["data"]: assert "email" in item diff --git a/src/docker-compose.override.yml b/src/docker-compose.override.yml index 469caed772..0c20cce54b 100644 --- a/src/docker-compose.override.yml +++ b/src/docker-compose.override.yml @@ -28,6 +28,10 @@ services: - traefik.http.routers.${STACK_NAME?Variable not set}-traefik-public-http.rule=Host(`${DOMAIN?Variable not set}`) - traefik.http.services.${STACK_NAME?Variable not set}-traefik-public.loadbalancer.server.port=80 + db: + ports: + - "5432:5432" + pgadmin: ports: - "5050:5050" @@ -84,14 +88,14 @@ services: - traefik.http.routers.${STACK_NAME?Variable not set}-frontend-http.rule=Host(`old-frontend.localhost.tiangolo.com`) - traefik.http.services.${STACK_NAME?Variable not set}-frontend.loadbalancer.server.port=80 - new-frontend: - build: - context: ./new-frontend - labels: - - traefik.enable=true - - traefik.constraint-label-stack=${TRAEFIK_TAG?Variable not set} - - traefik.http.routers.${STACK_NAME?Variable not set}-new-frontend-http.rule=PathPrefix(`/`) - - traefik.http.services.${STACK_NAME?Variable not set}-new-frontend.loadbalancer.server.port=80 +# new-frontend: +# build: +# context: ./new-frontend +# labels: +# - traefik.enable=true +# - traefik.constraint-label-stack=${TRAEFIK_TAG?Variable not set} +# - traefik.http.routers.${STACK_NAME?Variable not set}-new-frontend-http.rule=PathPrefix(`/`) +# - traefik.http.services.${STACK_NAME?Variable not set}-new-frontend.loadbalancer.server.port=80 networks: traefik-public: diff --git a/src/docker-compose.yml b/src/docker-compose.yml index 7c8d1f4e26..39cccb0fb8 100644 --- a/src/docker-compose.yml +++ b/src/docker-compose.yml @@ -191,16 +191,16 @@ services: - traefik.http.routers.${STACK_NAME?Variable not set}-frontend-http.rule=PathPrefix(`/`) - traefik.http.services.${STACK_NAME?Variable not set}-frontend.loadbalancer.server.port=80 - new-frontend: - image: '${DOCKER_IMAGE_NEW_FRONTEND?Variable not set}:${TAG-latest}' - build: - context: ./new-frontend - deploy: - labels: - - traefik.enable=true - - traefik.constraint-label-stack=${TRAEFIK_TAG?Variable not set} - - traefik.http.routers.${STACK_NAME?Variable not set}-new-frontend-http.rule=PathPrefix(`/`) - - traefik.http.services.${STACK_NAME?Variable not set}-new-frontend.loadbalancer.server.port=80 +# new-frontend: +# image: '${DOCKER_IMAGE_NEW_FRONTEND?Variable not set}:${TAG-latest}' +# build: +# context: ./new-frontend +# deploy: +# labels: +# - traefik.enable=true +# - traefik.constraint-label-stack=${TRAEFIK_TAG?Variable not set} +# - traefik.http.routers.${STACK_NAME?Variable not set}-new-frontend-http.rule=PathPrefix(`/`) +# - traefik.http.services.${STACK_NAME?Variable not set}-new-frontend.loadbalancer.server.port=80 volumes: From 5c4bc949c7d3829ecec63652fc5159f0dfa0fbf6 Mon Sep 17 00:00:00 2001 From: Esteban Maya Cadavid Date: Tue, 20 Feb 2024 10:47:36 -0500 Subject: [PATCH 2/6] :bug: FIX: Celery test and bcrypt version bug with passlib --- src/backend/app/app/tests/api/api_v1/test_celery.py | 4 ++-- src/backend/app/pyproject.toml | 2 +- src/docker-compose.override.yml | 8 ++++++++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/backend/app/app/tests/api/api_v1/test_celery.py b/src/backend/app/app/tests/api/api_v1/test_celery.py index 7b10a331f2..80973064ff 100644 --- a/src/backend/app/app/tests/api/api_v1/test_celery.py +++ b/src/backend/app/app/tests/api/api_v1/test_celery.py @@ -8,11 +8,11 @@ def test_celery_worker_test( client: TestClient, superuser_token_headers: Dict[str, str] ) -> None: - data = {"msg": "test"} + data = {"message": "test"} r = client.post( f"{settings.API_V1_STR}/utils/test-celery/", json=data, headers=superuser_token_headers, ) response = r.json() - assert response["msg"] == "Word received" + assert response["message"] == "Word received" diff --git a/src/backend/app/pyproject.toml b/src/backend/app/pyproject.toml index 649ea0f1d4..be38bb6b90 100644 --- a/src/backend/app/pyproject.toml +++ b/src/backend/app/pyproject.toml @@ -11,7 +11,7 @@ fastapi = "^0.104.1" python-multipart = "^0.0.6" email-validator = "^2.1.0.post1" celery = "^5.3.5" -passlib = {extras = ["bcrypt"], version = "^1.7.4"} +passlib = {extras = ["bcrypt"], version = "4.0.1"} tenacity = "^8.2.3" pydantic = "<2.0" emails = "^0.6" diff --git a/src/docker-compose.override.yml b/src/docker-compose.override.yml index 0c20cce54b..256a684d26 100644 --- a/src/docker-compose.override.yml +++ b/src/docker-compose.override.yml @@ -36,6 +36,14 @@ services: ports: - "5050:5050" + # Uncomment the section below to be able to debug locally + # queue: + # ports: + # - "5671:5671" + # - "5672:5672" + # - "15672:15672" + # - "15671:15671" + flower: ports: - "5555:5555" From 3569b91a9673a1e0376fa5370c5b05cc622aabe2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Sun, 25 Feb 2024 15:38:28 +0100 Subject: [PATCH 3/6] =?UTF-8?q?=F0=9F=91=B7=20Tweak=20CI=20to=20get=20clos?= =?UTF-8?q?er=20to=20the=20older=20GitLab=20CI,=20wait=20for=20services=20?= =?UTF-8?q?to=20be=20available?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/test.yaml | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 38e822af5f..7f3818e682 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,30 +1,37 @@ -name: Pytest Backend +name: Test on: push: branches: - master + pull_request: + types: + - opened + - synchronize jobs: - pytest: + test: runs-on: ubuntu-latest defaults: run: working-directory: src steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 - - name: Setup python 3.10 - uses: actions/setup-python@v2 + - name: Set up Python + uses: actions/setup-python@v5 with: python-version: '3.10' - - name: docker-compose setup - run: | - docker-compose up -d - - - name: docker-compose run pytest - run: | - docker exec src_backend_1 pytest --cov=app --cov-report=term-missing app/tests + - name: Docker Compose build + run: docker compose build + - name: Docker Compose remove old containers and volumes + run: docker compose down -v --remove-orphans + - name: Docker Compose up + run: docker compose up -d + - name: Docker Compose run tests + run: exec -T backend bash /app/tests-start.sh + - name: Docker Compose cleanup + run: docker compose down -v --remove-orphans From 066f0bbff40964263695d8f5dfa4e739122a470e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Sun, 25 Feb 2024 15:38:57 +0100 Subject: [PATCH 4/6] =?UTF-8?q?=F0=9F=93=8C=20Fix=20pin=20for=20bcrypt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/app/pyproject.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/backend/app/pyproject.toml b/src/backend/app/pyproject.toml index b4b94bfb32..e15178463d 100644 --- a/src/backend/app/pyproject.toml +++ b/src/backend/app/pyproject.toml @@ -11,7 +11,7 @@ fastapi = "^0.104.1" python-multipart = "^0.0.6" email-validator = "^2.1.0.post1" celery = "^5.3.5" -passlib = {extras = ["bcrypt"], version = "4.0.1"} +passlib = {extras = ["bcrypt"], version = "^1.7.4"} tenacity = "^8.2.3" pydantic = "<2.0" emails = "^0.6" @@ -23,6 +23,8 @@ python-jose = {extras = ["cryptography"], version = "^3.3.0"} httpx = "^0.25.1" psycopg = {extras = ["binary"], version = "^3.1.13"} sqlmodel = "^0.0.16" +# Pin bcrypt until passlib supports the latest +bcrypt = "4.0.1" [tool.poetry.group.dev.dependencies] mypy = "^1.7.0" From e9a3f3b3e9de2625ee5f987151411311e65cbb02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Sun, 25 Feb 2024 15:39:59 +0100 Subject: [PATCH 5/6] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Refactor=20models,=20m?= =?UTF-8?q?ake=20them=20all=20inherit=20from=20SQLModel=20for=20consistenc?= =?UTF-8?q?y?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/app/app/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/app/app/models.py b/src/backend/app/app/models.py index 56c6f3e82e..5331a96b1f 100644 --- a/src/backend/app/app/models.py +++ b/src/backend/app/app/models.py @@ -51,7 +51,7 @@ class UserOut(UserBase): id: int -class UsersOut(BaseModel): +class UsersOut(SQLModel): data: list[UserOut] count: int @@ -88,7 +88,7 @@ class ItemOut(ItemBase): owner_id: int -class ItemsOut(BaseModel): +class ItemsOut(SQLModel): data: list[ItemOut] count: int From 822b8e91ba66bc3b78add0a877127b7c97aa058d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Sun, 25 Feb 2024 15:42:41 +0100 Subject: [PATCH 6/6] =?UTF-8?q?=F0=9F=92=9A=20Fix=20CI=20command?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 7f3818e682..6cf1d1493d 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -32,6 +32,6 @@ jobs: - name: Docker Compose up run: docker compose up -d - name: Docker Compose run tests - run: exec -T backend bash /app/tests-start.sh + run: docker compose exec -T backend bash /app/tests-start.sh - name: Docker Compose cleanup run: docker compose down -v --remove-orphans