From d615680e913fb529e43b92bfcb56dca0dd267e91 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Wed, 8 Dec 2021 12:19:32 +0800 Subject: [PATCH 01/52] feat: person api --- pol/api/v0/models.py | 20 +++++++++++++++----- pol/api/v0/person.py | 31 +++++++++++++++++++++++++++++-- pol/curd/__init__.py | 22 +++++++++++++++++++--- pol/db_models/sa.py | 2 ++ 4 files changed, 65 insertions(+), 10 deletions(-) diff --git a/pol/api/v0/models.py b/pol/api/v0/models.py index a54d9dd41..aa65d05c2 100644 --- a/pol/api/v0/models.py +++ b/pol/api/v0/models.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional from pydantic import Field, BaseModel @@ -18,10 +18,20 @@ class Person(BaseModel): name: str type: int infobox: str + role: PersonRole + summary: str + subjects: List[int] + wiki: Optional[Dict[str, Any]] = Field( None, - description="server parsed infobox, a map from key to string or tuple", + description="server parsed infobox, a map from key to string or tuple\n" + "null if server infobox is not valid", ) - role: PersonRole - summary: str - img: Optional[str] + + img: Optional[str] = None + + gender: Optional[int] = Field(None, description="parsed from wiki, maybe null") + blood_type: Optional[int] = Field(None, description="parsed from wiki, maybe null") + birth_year: Optional[int] = Field(None, description="parsed from wiki, maybe null") + birth_mon: Optional[int] = Field(None, description="parsed from wiki, maybe null") + birth_day: Optional[int] = Field(None, description="parsed from wiki, maybe null") diff --git a/pol/api/v0/person.py b/pol/api/v0/person.py index e9f185b53..f43a4838d 100644 --- a/pol/api/v0/person.py +++ b/pol/api/v0/person.py @@ -7,7 +7,8 @@ from pol.api.v0 import models from pol.models import ErrorDetail from pol.depends import get_db -from pol.db.tables import ChiiPerson +from pol.db.tables import ChiiPerson, ChiiPersonField, ChiiPersonCsIndex +from pol.db_models import sa from pol.curd.exceptions import NotFoundError router = APIRouter() @@ -20,7 +21,7 @@ 404: res.response(model=ErrorDetail), }, ) -async def bgm_ip_map( +async def get_person( person_id: int, db: Database = Depends(get_db), ): @@ -32,6 +33,24 @@ async def bgm_ip_map( if person.prsn_redirect: return RedirectResponse(str(person.prsn_redirect)) + query = ( + sa.select([ChiiPersonCsIndex.subject_id]) + .where(ChiiPersonCsIndex.prsn_id == person_id) + .distinct(ChiiPersonCsIndex.subject_id) + .order_by(ChiiPersonCsIndex.subject_id) + ) + + result = [] + for r in await db.fetch_all(query): + result.append(ChiiPersonCsIndex(**r).subject_id) + + field = await curd.get_one_null( + db, + ChiiPersonField, + ChiiPersonField.prsn_id == person_id, + ChiiPersonField.prsn_cat == "prsn", + ) + m = models.Person( id=person.prsn_id, name=person.prsn_name, @@ -40,6 +59,7 @@ async def bgm_ip_map( role=person.role(), summary=person.prsn_summary, img=imgUrl(person.prsn_img), + subjects=result, ) try: @@ -47,4 +67,11 @@ async def bgm_ip_map( except wiki.WikiSyntaxError: pass + if field: + m.gender = field.gender + m.blood_type = field.bloodtype + m.birth_year = field.birth_year + m.birth_mon = field.birth_mon + m.birth_day = field.birth_day + return m diff --git a/pol/curd/__init__.py b/pol/curd/__init__.py index 3273abab8..78cabb9d7 100644 --- a/pol/curd/__init__.py +++ b/pol/curd/__init__.py @@ -1,4 +1,4 @@ -from typing import Type, TypeVar +from typing import List, Type, TypeVar, Optional from databases import Database from sqlalchemy.orm import DeclarativeMeta @@ -10,8 +10,8 @@ T = TypeVar("T", bound=DeclarativeMeta) -async def get_one(db: Database, t: Type[T], where) -> T: - query = sa.select(t).where(where).limit(1) +async def get_one(db: Database, t: Type[T], *where) -> T: + query = sa.select(t).where(*where).limit(1) r = await db.fetch_one(query) if r: @@ -20,4 +20,20 @@ async def get_one(db: Database, t: Type[T], where) -> T: raise NotFoundError() +async def get_one_null(db: Database, t: Type[T], *where) -> Optional[T]: + query = sa.select(t).where(*where).limit(1) + r = await db.fetch_one(query) + + if r: + return t(**r) + + +async def get_all(db: Database, t: Type[T], *where) -> List[T]: + query = sa.select(t).where(*where).limit(1) + result = [] + for r in await db.fetch_all(query): + result.append(**r) + return result + + __all__ = ["person", "get_one"] diff --git a/pol/db_models/sa.py b/pol/db_models/sa.py index da868fe05..ecf8afb90 100644 --- a/pol/db_models/sa.py +++ b/pol/db_models/sa.py @@ -4,6 +4,7 @@ Column, String, DateTime, + and_, func, join, text, @@ -24,4 +25,5 @@ "select", "update", "insert", + "and_", ] From 3954dc74305dd1682f0b39a2f111890b7a12b85b Mon Sep 17 00:00:00 2001 From: Trim21 Date: Wed, 8 Dec 2021 12:25:02 +0800 Subject: [PATCH 02/52] fix --- pol/curd/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pol/curd/__init__.py b/pol/curd/__init__.py index 78cabb9d7..509180295 100644 --- a/pol/curd/__init__.py +++ b/pol/curd/__init__.py @@ -32,7 +32,7 @@ async def get_all(db: Database, t: Type[T], *where) -> List[T]: query = sa.select(t).where(*where).limit(1) result = [] for r in await db.fetch_all(query): - result.append(**r) + result.append(t(**r)) return result From fea96cb6d7c8c14bdf26a55487b2e42dba45ed17 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Wed, 8 Dec 2021 12:25:31 +0800 Subject: [PATCH 03/52] mypy --- pol/api/v0/person.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pol/api/v0/person.py b/pol/api/v0/person.py index f43a4838d..a4e45c5b2 100644 --- a/pol/api/v0/person.py +++ b/pol/api/v0/person.py @@ -1,3 +1,5 @@ +from typing import List + from fastapi import Depends, APIRouter, HTTPException from databases import Database from starlette.responses import RedirectResponse @@ -40,7 +42,7 @@ async def get_person( .order_by(ChiiPersonCsIndex.subject_id) ) - result = [] + result: List[ChiiPersonCsIndex] = [] for r in await db.fetch_all(query): result.append(ChiiPersonCsIndex(**r).subject_id) From c5217565ae0315f4d9c305a7218ea1d6628a8f3b Mon Sep 17 00:00:00 2001 From: Trim21 Date: Wed, 8 Dec 2021 12:26:33 +0800 Subject: [PATCH 04/52] make mypy happy --- pol/api/v0/person.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pol/api/v0/person.py b/pol/api/v0/person.py index a4e45c5b2..7ee35fad9 100644 --- a/pol/api/v0/person.py +++ b/pol/api/v0/person.py @@ -42,7 +42,7 @@ async def get_person( .order_by(ChiiPersonCsIndex.subject_id) ) - result: List[ChiiPersonCsIndex] = [] + result: List[int] = [] for r in await db.fetch_all(query): result.append(ChiiPersonCsIndex(**r).subject_id) From 34c6c6945d2fe070b222ddbf4e45fd0c1c0d0a71 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Wed, 8 Dec 2021 12:29:50 +0800 Subject: [PATCH 05/52] mypy --- pol/api/v0/person.py | 25 +++++++++++++------------ pol/curd/__init__.py | 2 ++ 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/pol/api/v0/person.py b/pol/api/v0/person.py index 7ee35fad9..30d4e7ee9 100644 --- a/pol/api/v0/person.py +++ b/pol/api/v0/person.py @@ -46,13 +46,6 @@ async def get_person( for r in await db.fetch_all(query): result.append(ChiiPersonCsIndex(**r).subject_id) - field = await curd.get_one_null( - db, - ChiiPersonField, - ChiiPersonField.prsn_id == person_id, - ChiiPersonField.prsn_cat == "prsn", - ) - m = models.Person( id=person.prsn_id, name=person.prsn_name, @@ -65,15 +58,23 @@ async def get_person( ) try: - m.wiki = wiki.parse(person.prsn_infobox).info - except wiki.WikiSyntaxError: - pass - - if field: + field = await curd.get_one( + db, + ChiiPersonField, + ChiiPersonField.prsn_id == person_id, + ChiiPersonField.prsn_cat == "prsn", + ) m.gender = field.gender m.blood_type = field.bloodtype m.birth_year = field.birth_year m.birth_mon = field.birth_mon m.birth_day = field.birth_day + except NotFoundError: + pass + + try: + m.wiki = wiki.parse(person.prsn_infobox).info + except wiki.WikiSyntaxError: + pass return m diff --git a/pol/curd/__init__.py b/pol/curd/__init__.py index 509180295..fb19340e0 100644 --- a/pol/curd/__init__.py +++ b/pol/curd/__init__.py @@ -27,6 +27,8 @@ async def get_one_null(db: Database, t: Type[T], *where) -> Optional[T]: if r: return t(**r) + return None + async def get_all(db: Database, t: Type[T], *where) -> List[T]: query = sa.select(t).where(*where).limit(1) From 6843dd3e5fee023a7e6a13dc40e58a257e76bf2d Mon Sep 17 00:00:00 2001 From: Trim21 Date: Wed, 8 Dec 2021 12:31:48 +0800 Subject: [PATCH 06/52] test --- tests/app/api_v0.py | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 tests/app/api_v0.py diff --git a/tests/app/api_v0.py b/tests/app/api_v0.py new file mode 100644 index 000000000..6167c3be4 --- /dev/null +++ b/tests/app/api_v0.py @@ -0,0 +1,11 @@ +from starlette.testclient import TestClient + + +def test_openapi_json(client: TestClient): + response = client.get("/api/v0/person/2") + assert response.status_code == 200 + assert response.headers["content-type"] == "application/json" + + res = response.json() + assert res["img"] is None + assert isinstance(res["subjects"], list) From fd7304af29ff7de2b190abbd21fe647fb6039f92 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Wed, 8 Dec 2021 12:40:02 +0800 Subject: [PATCH 07/52] add test case --- pol/api/v0/models.py | 1 + pol/api/v0/person.py | 1 + tests/app/api_v0.py | 21 ++++++++++++++++++--- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/pol/api/v0/models.py b/pol/api/v0/models.py index aa65d05c2..304b4804b 100644 --- a/pol/api/v0/models.py +++ b/pol/api/v0/models.py @@ -21,6 +21,7 @@ class Person(BaseModel): role: PersonRole summary: str subjects: List[int] + locked: bool wiki: Optional[Dict[str, Any]] = Field( None, diff --git a/pol/api/v0/person.py b/pol/api/v0/person.py index 30d4e7ee9..60597ee26 100644 --- a/pol/api/v0/person.py +++ b/pol/api/v0/person.py @@ -55,6 +55,7 @@ async def get_person( summary=person.prsn_summary, img=imgUrl(person.prsn_img), subjects=result, + locked=person.prsn_lock, ) try: diff --git a/tests/app/api_v0.py b/tests/app/api_v0.py index 6167c3be4..04cf74756 100644 --- a/tests/app/api_v0.py +++ b/tests/app/api_v0.py @@ -1,11 +1,26 @@ from starlette.testclient import TestClient -def test_openapi_json(client: TestClient): +def test_person_basic(client: TestClient): response = client.get("/api/v0/person/2") assert response.status_code == 200 assert response.headers["content-type"] == "application/json" + data = response.json() + assert data["img"] is None + assert isinstance(data["subjects"], list) + assert not data["lock"] + + +def test_person_redirect(client: TestClient): + response = client.get("/api/v0/person/10", allow_redirects=False) + assert response.status_code == 302 + + +def test_person_lock(client: TestClient): + response = client.get("/api/v0/person/9") + assert response.status_code == 200 + assert response.headers["content-type"] == "application/json" + res = response.json() - assert res["img"] is None - assert isinstance(res["subjects"], list) + assert res["lock"] From 4bd905653e1cdce7e827b3ec7ead683d72f19121 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Wed, 8 Dec 2021 12:56:52 +0800 Subject: [PATCH 08/52] make sure test run --- pol/api/v0/person.py | 4 +++- .../app/{api_v0.py => api_v0/test_person.py} | 19 ++++++++++++++++--- 2 files changed, 19 insertions(+), 4 deletions(-) rename tests/app/{api_v0.py => api_v0/test_person.py} (54%) diff --git a/pol/api/v0/person.py b/pol/api/v0/person.py index 60597ee26..d237eab64 100644 --- a/pol/api/v0/person.py +++ b/pol/api/v0/person.py @@ -38,10 +38,12 @@ async def get_person( query = ( sa.select([ChiiPersonCsIndex.subject_id]) .where(ChiiPersonCsIndex.prsn_id == person_id) - .distinct(ChiiPersonCsIndex.subject_id) + .distinct() .order_by(ChiiPersonCsIndex.subject_id) ) + print(str(query)) + result: List[int] = [] for r in await db.fetch_all(query): result.append(ChiiPersonCsIndex(**r).subject_id) diff --git a/tests/app/api_v0.py b/tests/app/api_v0/test_person.py similarity index 54% rename from tests/app/api_v0.py rename to tests/app/api_v0/test_person.py index 04cf74756..ba20e320c 100644 --- a/tests/app/api_v0.py +++ b/tests/app/api_v0/test_person.py @@ -1,6 +1,18 @@ from starlette.testclient import TestClient +def test_person_not_found(client: TestClient): + response = client.get("/api/v0/person/2000000") + assert response.status_code == 404 + assert response.headers["content-type"] == "application/json" + + +def test_person_not_valid(client: TestClient): + response = client.get("/api/v0/person/hello") + assert response.status_code == 422 + assert response.headers["content-type"] == "application/json" + + def test_person_basic(client: TestClient): response = client.get("/api/v0/person/2") assert response.status_code == 200 @@ -9,12 +21,13 @@ def test_person_basic(client: TestClient): data = response.json() assert data["img"] is None assert isinstance(data["subjects"], list) - assert not data["lock"] + assert not data["locked"] + assert len(set(data["subjects"])) == len(data["subjects"]) def test_person_redirect(client: TestClient): response = client.get("/api/v0/person/10", allow_redirects=False) - assert response.status_code == 302 + assert response.status_code == 307 def test_person_lock(client: TestClient): @@ -23,4 +36,4 @@ def test_person_lock(client: TestClient): assert response.headers["content-type"] == "application/json" res = response.json() - assert res["lock"] + assert res["locked"] From ec138276df0078aad92c31ac4103547ee47c6deb Mon Sep 17 00:00:00 2001 From: Trim21 Date: Wed, 8 Dec 2021 13:09:57 +0800 Subject: [PATCH 09/52] check person id range --- pol/api/v0/person.py | 4 ++-- pol/curd/__init__.py | 20 +------------------- tests/app/api_v0/test_person.py | 4 ++++ 3 files changed, 7 insertions(+), 21 deletions(-) diff --git a/pol/api/v0/person.py b/pol/api/v0/person.py index d237eab64..48f0218b6 100644 --- a/pol/api/v0/person.py +++ b/pol/api/v0/person.py @@ -1,6 +1,6 @@ from typing import List -from fastapi import Depends, APIRouter, HTTPException +from fastapi import Path, Depends, APIRouter, HTTPException from databases import Database from starlette.responses import RedirectResponse @@ -24,7 +24,7 @@ }, ) async def get_person( - person_id: int, + person_id: int = Path(..., gt=0), db: Database = Depends(get_db), ): try: diff --git a/pol/curd/__init__.py b/pol/curd/__init__.py index fb19340e0..b6128feb7 100644 --- a/pol/curd/__init__.py +++ b/pol/curd/__init__.py @@ -1,4 +1,4 @@ -from typing import List, Type, TypeVar, Optional +from typing import Type, TypeVar from databases import Database from sqlalchemy.orm import DeclarativeMeta @@ -20,22 +20,4 @@ async def get_one(db: Database, t: Type[T], *where) -> T: raise NotFoundError() -async def get_one_null(db: Database, t: Type[T], *where) -> Optional[T]: - query = sa.select(t).where(*where).limit(1) - r = await db.fetch_one(query) - - if r: - return t(**r) - - return None - - -async def get_all(db: Database, t: Type[T], *where) -> List[T]: - query = sa.select(t).where(*where).limit(1) - result = [] - for r in await db.fetch_all(query): - result.append(t(**r)) - return result - - __all__ = ["person", "get_one"] diff --git a/tests/app/api_v0/test_person.py b/tests/app/api_v0/test_person.py index ba20e320c..6a82bdbc0 100644 --- a/tests/app/api_v0/test_person.py +++ b/tests/app/api_v0/test_person.py @@ -12,6 +12,10 @@ def test_person_not_valid(client: TestClient): assert response.status_code == 422 assert response.headers["content-type"] == "application/json" + response = client.get("/api/v0/person/0") + assert response.status_code == 422 + assert response.headers["content-type"] == "application/json" + def test_person_basic(client: TestClient): response = client.get("/api/v0/person/2") From 450fd73b8917ffd664f73f3c592778d18915a1c9 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Wed, 8 Dec 2021 16:45:50 +0800 Subject: [PATCH 10/52] omit empty --- pol/api/v0/person.py | 12 +++++------- pol/server.py | 2 +- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/pol/api/v0/person.py b/pol/api/v0/person.py index 48f0218b6..49f0caed4 100644 --- a/pol/api/v0/person.py +++ b/pol/api/v0/person.py @@ -42,8 +42,6 @@ async def get_person( .order_by(ChiiPersonCsIndex.subject_id) ) - print(str(query)) - result: List[int] = [] for r in await db.fetch_all(query): result.append(ChiiPersonCsIndex(**r).subject_id) @@ -67,11 +65,11 @@ async def get_person( ChiiPersonField.prsn_id == person_id, ChiiPersonField.prsn_cat == "prsn", ) - m.gender = field.gender - m.blood_type = field.bloodtype - m.birth_year = field.birth_year - m.birth_mon = field.birth_mon - m.birth_day = field.birth_day + m.gender = field.gender or None + m.blood_type = field.bloodtype or None + m.birth_year = field.birth_year or None + m.birth_mon = field.birth_mon or None + m.birth_day = field.birth_day or None except NotFoundError: pass diff --git a/pol/server.py b/pol/server.py index b62de06b9..5d2a971ea 100644 --- a/pol/server.py +++ b/pol/server.py @@ -52,7 +52,7 @@ async def doc(): - + Pol server - Swagger UI