Merge pull request #1 from sinisaos/cursor_pagination

add cursor pagination

Author: sinisaos

.github/workflows/release.yaml

@@ -0,0 +1,24 @@
+name: Release
+
+on:
+  release:
+    types:
+      - created
+
+jobs:
+  release:
+    name: "Publish release"
+    runs-on: "ubuntu-latest"
+
+    steps:
+      - uses: "actions/checkout@v2"
+      - uses: "actions/setup-python@v1"
+        with:
+          python-version: 3.9
+      - name: "Install dependencies"
+        run: "pip install -r requirements/dev-requirements.txt"
+      - name: "Publish to PyPI"
+        run: "./scripts/release.sh"
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}

README.md

@@ -1,2 +1,114 @@
-# piccolo_cursor_pagination
-Cursor pagination for Piccolo ORM
+## Cursor pagination for Piccolo ORM
+
+[Piccolo](https://github.com/piccolo-orm) is an great ecosystem that helps you create [ASGI](https://asgi.readthedocs.io/en/latest/) apps faster and easier. [LimitOffset](https://piccolo-api.readthedocs.io/en/latest/crud/piccolo_crud.html#pagination) is the default Piccolo pagination used by Piccolo Admin and Piccolo API. This package contains usage of cursor pagination which is suitable for large data sets and has better performance than ``LimitOffset`` pagination,
+but it is **strictly optional** because it does not work with the Piccolo Admin and Piccolo API.
+
+# Installation 
+
+```bash
+pip install piccolo_cursor_pagination
+```
+# Usage
+
+Example usage of ``CursorPagination``:
+
+```python
+import typing as t
+
+from fastapi import FastAPI, Request
+from fastapi.responses import JSONResponse
+from piccolo_api.crud.serializers import create_pydantic_model
+from piccolo_cursor_pagination.pagination import CursorPagination
+
+from home.tables import Task
+
+app = FastAPI()
+
+TaskModelOut: t.Any = create_pydantic_model(
+    table=Task, include_default_columns=True, model_name="TaskModelOut"
+)
+
+@app.get("/tasks/", response_model=t.List[TaskModelOut])
+async def tasks(
+    request: Request,
+    __cursor: str,
+    __previous: t.Optional[str] = None,
+):
+    try:
+        cursor = request.query_params["__cursor"]
+        previous = request.query_params["__previous"]
+        paginator = CursorPagination(cursor=cursor)
+        rows_result, headers_result = paginator.get_cursor_rows(Task, request)
+        rows = await rows_result.run()
+        headers = headers_result
+        response = JSONResponse(
+            {"rows": rows[::-1]},
+            headers={
+                "next_cursor": headers["cursor"],
+            },
+        )
+    except KeyError:
+        cursor = request.query_params["__cursor"]
+        paginator = CursorPagination(cursor=cursor)
+        rows_result, headers_result = paginator.get_cursor_rows(Task, request)
+        rows = await rows_result.run()
+        headers = headers_result
+        response = JSONResponse(
+            {"rows": rows},
+            headers={
+                "next_cursor": headers["cursor"],
+            },
+        )
+    return response
+
+@app.on_event("startup")
+async def open_database_connection_pool():
+    try:
+        engine = engine_finder()
+        await engine.start_connection_pool()
+    except Exception:
+        print("Unable to connect to the database")
+
+
+@app.on_event("shutdown")
+async def close_database_connection_pool():
+    try:
+        engine = engine_finder()
+        await engine.close_connection_pool()
+    except Exception:
+        print("Unable to connect to the database")
+```
+The ``CursorPagination`` stores the value of ``next_cursor`` in the response headers. 
+We can then use the ``next_cursor`` value to get new set of results by passing 
+``next_cursor`` to ``__cursor`` query parameter.
+
+Full Piccolo ASGI app is in **example** folder.
+
+# Customization
+
+The ``CursorPagination`` class has a default value of ``page_size`` and ``order_by``, 
+but we can overide this value in constructor to adjust the way the results are displayed.
+
+Example of displaying results in ascending order and page size is 10:
+
+```python
+paginator = CursorPagination(cursor=cursor, page_size=10, order_by="id")
+```
+
+# Directions
+
+The ``CursorPagination`` has the ability to move forward and backward. 
+To go backward we have to pass ``__previous=yes`` in the query parameters.
+
+Example usage of direction:
+
+```
+GET http://localhost:8000/tasks/?__cursor=NA== (forward)
+GET http://localhost:8000/tasks/?__cursor=NA==&__previous=yes (backward)
+```
+
+# Limitations
+
+You need to be aware that cursor pagination has several trade offs. The cursor must be based on a unique and sequential column in the table and the client can't go to a specific page because there is no concept of the total number of pages or results.
+
+> **WARNING**: ``CursorPagination`` use Piccolo ORM ``id`` (PK type **integer**) column as unique and sequential column for pagination.

example/README.md

@@ -0,0 +1,21 @@
+# piccolo_project
+
+## Setup
+
+### Install requirements
+
+```bash
+pip install -r requirements.txt
+```
+
+### Getting started guide
+
+```bash
+python main.py
+```
+
+### Running tests
+
+```bash
+piccolo tester run
+```

example/__init__.py

@@ -0,0 +1,122 @@
+import typing as t
+
+from fastapi import FastAPI, Request
+from fastapi.responses import JSONResponse
+from piccolo_admin.endpoints import create_admin
+from piccolo_api.crud.serializers import create_pydantic_model
+from piccolo_cursor_pagination.pagination import CursorPagination
+from piccolo.engine import engine_finder
+from starlette.routing import Route, Mount
+from starlette.staticfiles import StaticFiles
+
+from home.endpoints import HomeEndpoint
+from home.piccolo_app import APP_CONFIG
+from home.tables import Task
+
+
+app = FastAPI(
+    routes=[
+        Route("/", HomeEndpoint),
+        Mount(
+            "/admin/",
+            create_admin(
+                tables=APP_CONFIG.table_classes,
+                # Required when running under HTTPS:
+                # allowed_hosts=['my_site.com']
+            ),
+        ),
+        Mount("/static/", StaticFiles(directory="static")),
+    ],
+)
+
+
+TaskModelIn: t.Any = create_pydantic_model(
+    table=Task, model_name="TaskModelIn"
+)
+TaskModelOut: t.Any = create_pydantic_model(
+    table=Task, include_default_columns=True, model_name="TaskModelOut"
+)
+
+
+@app.get("/tasks/", response_model=t.List[TaskModelOut])
+async def tasks(
+    request: Request,
+    __cursor: str,
+    __previous: t.Optional[str] = None,
+):
+    try:
+        cursor = request.query_params["__cursor"]
+        previous = request.query_params["__previous"]
+        paginator = CursorPagination(cursor=cursor, page_size=1, order_by="id")
+        rows_result, headers_result = paginator.get_cursor_rows(Task, request)
+        rows = await rows_result.run()
+        headers = headers_result
+        response = JSONResponse(
+            {"rows": rows[::-1]},
+            headers={
+                "next_cursor": headers["cursor"],
+            },
+        )
+    except KeyError:
+        cursor = request.query_params["__cursor"]
+        paginator = CursorPagination(cursor=cursor, page_size=1, order_by="id")
+        rows_result, headers_result = paginator.get_cursor_rows(Task, request)
+        rows = await rows_result.run()
+        headers = headers_result
+        response = JSONResponse(
+            {"rows": rows},
+            headers={
+                "next_cursor": headers["cursor"],
+            },
+        )
+    return response
+
+
+@app.post("/tasks/", response_model=TaskModelOut)
+async def create_task(task_model: TaskModelIn):
+    task = Task(**task_model.dict())
+    await task.save()
+    return task.to_dict()
+
+
+@app.put("/tasks/{task_id}/", response_model=TaskModelOut)
+async def update_task(task_id: int, task_model: TaskModelIn):
+    task = await Task.objects().get(Task.id == task_id)
+    if not task:
+        return JSONResponse({}, status_code=404)
+
+    for key, value in task_model.dict().items():
+        setattr(task, key, value)
+
+    await task.save()
+
+    return task.to_dict()
+
+
+@app.delete("/tasks/{task_id}/")
+async def delete_task(task_id: int):
+    task = await Task.objects().get(Task.id == task_id)
+    if not task:
+        return JSONResponse({}, status_code=404)
+
+    await task.remove()
+
+    return JSONResponse({})
+
+
+@app.on_event("startup")
+async def open_database_connection_pool():
+    try:
+        engine = engine_finder()
+        await engine.start_connection_pool()
+    except Exception:
+        print("Unable to connect to the database")
+
+
+@app.on_event("shutdown")
+async def close_database_connection_pool():
+    try:
+        engine = engine_finder()
+        await engine.close_connection_pool()
+    except Exception:
+        print("Unable to connect to the database")

example/app.py

@@ -0,0 +1,17 @@
+import os
+import sys
+
+from piccolo.utils.warnings import colored_warning
+
+
+def pytest_configure(*args):
+    if os.environ.get("PICCOLO_TEST_RUNNER") != "True":
+        colored_warning(
+            "\n\n"
+            "We recommend running Piccolo tests using the "
+            "`piccolo tester run` command, which wraps Pytest, and makes "
+            "sure the test database is being used. "
+            "To stop this warning, modify conftest.py."
+            "\n\n"
+        )
+        sys.exit(1)

example/conftest.py

@@ -0,0 +1,23 @@
+import os
+
+import jinja2
+from starlette.endpoints import HTTPEndpoint
+from starlette.responses import HTMLResponse
+
+
+ENVIRONMENT = jinja2.Environment(
+    loader=jinja2.FileSystemLoader(
+        searchpath=os.path.join(os.path.dirname(__file__), "templates")
+    )
+)
+
+
+class HomeEndpoint(HTTPEndpoint):
+    async def get(self, request):
+        template = ENVIRONMENT.get_template("home.html.jinja")
+
+        content = template.render(
+            title="Piccolo + ASGI",
+        )
+
+        return HTMLResponse(content)

example/home/__init__.py

@@ -0,0 +1,20 @@
+"""
+Import all of the Tables subclasses in your app here, and register them with
+the APP_CONFIG.
+"""
+
+import os
+
+from piccolo.conf.apps import AppConfig, table_finder
+
+
+CURRENT_DIRECTORY = os.path.dirname(os.path.abspath(__file__))
+
+
+APP_CONFIG = AppConfig(
+    app_name="home",
+    migrations_folder_path=os.path.join(CURRENT_DIRECTORY, "piccolo_migrations"),
+    table_classes=table_finder(modules=["home.tables"], exclude_imported=True),
+    migration_dependencies=[],
+    commands=[],
+)

example/home/endpoints.py

@@ -0,0 +1 @@
+Add migrations using `piccolo migrations new home --auto`.