Updating to support categories

backend/Dockerfile

@@ -8,5 +8,9 @@ RUN pip install -r requirements.txt
 RUN apt update && apt install -y ffmpeg
 
 COPY . .
+
+# RUN the alembic migrations
+RUN alembic upgrade head
+
 CMD ["sh", "-c", "celery -A src.routers.signvideo worker --loglevel=INFO & python main.py"]
 # CMD ["python", "main.py"]

backend/alembic.ini

@@ -0,0 +1,113 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = alembic
+
+transactional_ddl = True
+
+# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
+# Uncomment the line below if you want the files to be prepended with date and time
+# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
+# for all available tokens
+# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
+
+# sys.path path, will be prepended to sys.path if present.
+# defaults to the current working directory.
+prepend_sys_path = .
+
+# timezone to use when rendering the date within the migration file
+# as well as the filename.
+# If specified, requires the python-dateutil library that can be
+# installed by adding `alembic[tz]` to the pip requirements
+# string value is passed to dateutil.tz.gettz()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the
+# "slug" field
+# truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; This defaults
+# to alembic/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path.
+# The path separator used here should be the separator specified by "version_path_separator" below.
+# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions
+
+# version path separator; As mentioned above, this is the character used to split
+# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
+# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas.
+# Valid values for version_path_separator are:
+#
+# version_path_separator = :
+# version_path_separator = ;
+# version_path_separator = space
+version_path_separator = os  # Use os.pathsep. Default configuration used for new projects.
+
+# set to 'true' to search source files recursively
+# in each "version_locations" directory
+# new in Alembic version 1.10
+# recursive_version_locations = false
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+sqlalchemy.url = driver://user:pass@localhost/dbname
+
+
+[post_write_hooks]
+# post_write_hooks defines scripts or Python functions that are run
+# on newly generated revision scripts.  See the documentation for further
+# detail and examples
+
+# format using "black" - use the console_scripts runner, against the "black" entrypoint
+# hooks = black
+# black.type = console_scripts
+# black.entrypoint = black
+# black.options = -l 79 REVISION_SCRIPT_FILENAME
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S
+

backend/alembic/README

@@ -0,0 +1 @@
+Generic single-database configuration.

backend/alembic/env.py

@@ -0,0 +1,81 @@
+import asyncio
+from logging.config import fileConfig
+
+from sqlalchemy import engine_from_config, pool
+from sqlmodel import Session, SQLModel, create_engine
+from sqlmodel.ext.asyncio.session import AsyncEngine
+
+from alembic import context
+from src.database.engine import engine
+from src.models import *  # necessarily to import something from file where your models are stored
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+target_metadata = None
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+target_metadata = None 
+# comment line above and instead of that write
+target_metadata = SQLModel.metadata
+
+
+async def run_migrations_offline() -> None:
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    connectable = engine
+
+    async with connectable.connect() as connection:
+        await connection.run_sync(do_run_migrations)
+
+def do_run_migrations(connection):
+    context.configure(connection=connection, target_metadata=target_metadata, render_as_batch=True)
+
+    with context.begin_transaction():
+
+        context.run_migrations()
+
+
+
+async def run_migrations_online() -> None:
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    connectable = engine
+
+    async with connectable.connect() as connection:
+        await connection.run_sync(do_run_migrations)
+
+if context.is_offline_mode():
+    asyncio.run(run_migrations_offline())
+else:
+    asyncio.run(run_migrations_online())

backend/alembic/script.py.mako

@@ -0,0 +1,25 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+import sqlmodel
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}

backend/alembic/versions/2d2d4523082b_add_category_table.py

@@ -0,0 +1,46 @@
+"""add category table
+
+Revision ID: 2d2d4523082b
+Revises: 
+Create Date: 2023-03-09 20:29:44.081609
+
+"""
+import sqlalchemy as sa
+import sqlmodel
+
+from alembic import op
+from src.models.category import Category
+
+# revision identifiers, used by Alembic.
+revision = '2d2d4523082b'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # create category table
+    Category.__table__.create(bind=op.get_bind())
+
+    op.bulk_insert(
+        Category.__table__,
+        [
+            {"id": 1, "name": "Alfabet"},
+        ]
+    )
+    
+    with op.batch_alter_table('sign', schema=None) as batch_op:
+        # create column category_id
+        batch_op.add_column(sa.Column('category_id', sa.Integer(), nullable=True, default=1))
+        batch_op.create_foreign_key("sign_category", 'category', ['category_id'], ['id'])
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('sign', schema=None) as batch_op:
+        batch_op.drop_constraint("sign_category", type_='foreignkey')
+        batch_op.drop_column('category_id')
+    # drop category table
+    Category.__table__.drop(bind=op.get_bind())
+
+    # ### end Alembic commands ###

backend/requirements.txt

@@ -11,4 +11,5 @@ Pillow
 joblib
 celery
 numpy
-redis
+redis
+alembic

backend/src/app.py

@@ -51,11 +51,13 @@ app.add_middleware(
 )
 
 # include the routers
-from .routers import auth_router, signs_router, signvideo_router
+from .routers import (auth_router, category_router, signs_router,
+                      signvideo_router)
 
 app.include_router(auth_router)
 app.include_router(signs_router)
 app.include_router(signvideo_router)
+app.include_router(category_router)
 
 # Add the exception handlers
 app.add_exception_handler(BaseException, base_exception_handler)

backend/src/models/SQLModelExtended.py

@@ -3,6 +3,7 @@ from typing import List, Optional
 from pydantic import BaseModel
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlmodel import SQLModel
+
 from src.database.crud import delete, read_all_where, read_where, update
 
 
@@ -34,8 +35,8 @@ class SQLModelExtended(SQLModel):
         return res
 
     @classmethod
-    async def get_all_where(self, *args, session: AsyncSession):
-        res = await read_all_where(self, *args, session=session)
+    async def get_all_where(self, *args, select_in_load: List = [], session: AsyncSession):
+        res = await read_all_where(self, *args, select_in_load=select_in_load, session=session)
         return res
 
     async def delete(self, session: AsyncSession) -> None:

backend/src/models/__init__.py

@@ -0,0 +1,5 @@
+import glob
+from os.path import basename, dirname, isfile, join
+
+modules = glob.glob(join(dirname(__file__), "*.py"))
+__all__ = [ basename(f)[:-3] for f in modules if isfile(f) and not f.endswith('__init__.py')]

backend/src/models/category.py

@@ -0,0 +1,54 @@
+from typing import List
+
+import numpy as np
+from pydantic import BaseModel
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlmodel import Field, Relationship
+
+from src.models.sign import Sign, SignOut
+from src.models.SQLModelExtended import SQLModelExtended
+
+
+class Category(SQLModelExtended, table=True):
+    id: int = Field(primary_key=True)
+
+    name: str = Field(unique=True)
+    enabled: bool = Field(default=True)
+
+    # list of signs that belong to this category
+    signs: List[Sign] = Relationship(
+        back_populates="category",
+        sa_relationship_kwargs={"lazy": "selectin"},
+    )
+
+    @classmethod
+    async def get_random_sign(self, session: AsyncSession):
+        # get all categories
+
+        categories = await self.get_all_where(Category.enabled==True, select_in_load=[Category.signs],session=session)
+
+        # get all signs in one list with list comprehension
+        signs = [s for c in categories for s in c.signs]
+
+        sign_videos = [len(s.sign_videos) for s in signs]
+
+        random_prob = [1 / (x + 1) for x in sign_videos]
+        random_prob = random_prob / np.sum(random_prob)
+
+        # get random sign
+        sign = np.random.choice(signs, p=random_prob)
+
+        return sign
+
+class CategoryPost(BaseModel):
+    name: str
+
+class CategoryPut(BaseModel):
+    id: int
+    name: str
+    enabled: bool = True
+
+class CategoryOut(BaseModel):
+    id: int
+    name: str
+    enabled: bool

backend/src/models/sign.py

@@ -3,7 +3,6 @@ from typing import List
 import numpy as np
 import requests
 from pydantic import BaseModel
-from sqlalchemy.ext.asyncio import AsyncSession
 from sqlmodel import Field, Relationship, SQLModel
 
 from src.exceptions.base_exception import BaseException
@@ -25,8 +24,12 @@ class Sign(SQLModelExtended, table=True):
         sa_relationship_kwargs={"lazy": "selectin"},
     )
 
-    def __init__(self, url):
+    category_id: int = Field(foreign_key="category.id")
+    category: "Category" = Relationship(back_populates="signs")
+
+    def __init__(self, url, category_id):
         self.url = url
+        self.category_id = category_id
 
         # get name and sign id from url
         try:
@@ -53,23 +56,6 @@ class Sign(SQLModelExtended, table=True):
         if self.video_url is None:
             raise BaseException(404, "Video url not found")
 
-    @classmethod
-    async def get_random(self, session: AsyncSession):
-        signs = await self.get_all(select_in_load=[Sign.sign_videos],session=session)
-
-        sign_videos = [len(s.sign_videos) for s in signs]
-
-        # get probability based on number of videos, lower must be more likely
-        # the sum must be 1
-
-        random_prob = [1 / (x + 1) for x in sign_videos]
-        random_prob = random_prob / np.sum(random_prob)
-
-        # get random sign
-        sign = np.random.choice(signs, p=random_prob)
-
-        return sign
-
 
 
 
@@ -79,6 +65,7 @@ class SignOut(BaseModel):
     name: str
     sign_id: str
     video_url: str
+    category_id: int
 
     sign_videos: List[SignVideo] = []
 
@@ -87,4 +74,5 @@ class SignOutSimple(BaseModel):
     url: str
     name: str
     sign_id: str
-    video_url: str
+    video_url: str
+    category_id: int

backend/src/routers/__init__.py

@@ -1,3 +1,4 @@
 from .auth import router as auth_router
+from .category import router as category_router
 from .signs import router as signs_router
 from .signvideo import router as signvideo_router

backend/src/routers/category.py

@@ -0,0 +1,106 @@
+from typing import List
+
+from fastapi import APIRouter, BackgroundTasks, Depends, status
+from fastapi_jwt_auth import AuthJWT
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from src.database.database import get_session
+from src.exceptions.base_exception import BaseException
+from src.exceptions.login_exception import LoginException
+from src.models.auth import User
+from src.models.category import (Category, CategoryOut, CategoryPost,
+                                 CategoryPut)
+from src.models.sign import Sign, SignOut
+
+router = APIRouter(prefix="/categories")
+
+@router.get("/", response_model=List[Category])
+async def get_categories(Authorize: AuthJWT = Depends(), session: AsyncSession = Depends(get_session)):
+    Authorize.jwt_required()
+    
+    user = Authorize.get_jwt_subject()
+    user = await User.get_by_id(id=user, session=session)
+
+    if not user:
+        raise LoginException("User not found")
+
+    categories = await Category.get_all(session=session, select_in_load=[Category.signs])
+    return categories
+
+@router.get("/{category_id}/signs", response_model=List[SignOut])
+async def get_signs_by_category(category_id: int, Authorize: AuthJWT = Depends(), session: AsyncSession = Depends(get_session)):
+    Authorize.jwt_required()
+
+    user = Authorize.get_jwt_subject()
+    user = await User.get_by_id(id=user, session=session)
+
+    if not user:
+        raise LoginException("User not found")
+    
+    signs = await Sign.get_all_where(Sign.category_id==category_id, select_in_load=[Sign.sign_videos], session=session)
+    return signs
+
+@router.post("/", status_code=status.HTTP_201_CREATED, response_model=CategoryOut)
+async def create_category(category: CategoryPost, Authorize: AuthJWT = Depends(), session: AsyncSession = Depends(get_session)):
+    Authorize.jwt_required()
+
+    user = Authorize.get_jwt_subject()
+    user = await User.get_by_id(id=user, session=session)
+
+    if not user:
+        raise LoginException("User not found")
+
+    # Category name cannot be empty or only exist of spaces
+    if not category.name or category.name.isspace():
+        raise BaseException(message="Category name cannot be empty", status_code=status.HTTP_400_BAD_REQUEST)
+
+    try:
+        c = Category(name=category.name)
+        await c.save(session=session)
+    except Exception as e:
+        raise BaseException(message="Category already exists", status_code=status.HTTP_400_BAD_REQUEST)
+    return c
+
+@router.put("/", response_model=CategoryOut)
+async def update_category(category: CategoryPut, Authorize: AuthJWT = Depends(), session: AsyncSession = Depends(get_session)):
+    Authorize.jwt_required()
+
+    user = Authorize.get_jwt_subject()
+    user = await User.get_by_id(id=user, session=session)
+
+    if not user:
+        raise LoginException("User not found")
+
+    # Category name cannot be empty or only exist of spaces
+    if not category.name or category.name.isspace():
+        raise BaseException(message="Category name cannot be empty", status_code=status.HTTP_400_BAD_REQUEST)
+
+    c = await Category.get_by_id(id=category.id, session=session)
+    if not c:
+        raise BaseException(message="Category not found", status_code=status.HTTP_404_NOT_FOUND)
+
+    c.name = category.name
+    c.enabled = category.enabled
+    await c.save(session=session)
+    return c
+
+@router.delete("/{category_id}")
+async def delete_category(category_id: int, Authorize: AuthJWT = Depends(), session: AsyncSession = Depends(get_session)):
+    Authorize.jwt_required()
+
+    user = Authorize.get_jwt_subject()
+    user = await User.get_by_id(id=user, session=session)
+
+    if not user:
+        raise LoginException("User not found")
+
+    c = await Category.get_by_id(id=category_id, session=session)
+    print(c)
+    if not c:
+        raise BaseException(message="Category not found", status_code=status.HTTP_404_NOT_FOUND)
+    
+    if (len(c.signs) > 0):
+        raise BaseException(message="Category cannot be deleted because it contains signs", status_code=status.HTTP_400_BAD_REQUEST)
+
+    await c.delete(session=session)
+    return {"message": "Category deleted successfully"}

backend/src/routers/signs.py

@@ -12,24 +12,25 @@ from sqlalchemy.ext.asyncio import AsyncSession
 import src.settings as settings
 from src.database.database import get_session
 from src.exceptions.login_exception import LoginException
-from src.models.auth import Login, User
+from src.models.auth import User
+from src.models.category import Category
 from src.models.sign import Sign, SignOut, SignOutSimple
 from src.models.signvideo import SignVideo
-from src.models.token import TokenExtended
 
 router = APIRouter(prefix="/signs")
 
-class SignUrl(BaseModel):
+class SignIn(BaseModel):
+    category: int
     url: str
 
 @router.get("/random", status_code=status.HTTP_200_OK, response_model=SignOutSimple)
 async def get_random_sign(session: AsyncSession = Depends(get_session)):
     # get a random sign where there is not much data from yet
-    sign = await Sign.get_random(session=session)
+    sign = await Category.get_random_sign(session=session)
     return sign
 
 @router.post("/", status_code=status.HTTP_201_CREATED, response_model=SignOut)
-async def add_sign(url: SignUrl, Authorize: AuthJWT = Depends(), session: AsyncSession = Depends(get_session)):
+async def add_sign(sign: SignIn, Authorize: AuthJWT = Depends(), session: AsyncSession = Depends(get_session)):
     Authorize.jwt_required()
 
     user = Authorize.get_jwt_subject()
@@ -38,7 +39,7 @@ async def add_sign(url: SignUrl, Authorize: AuthJWT = Depends(), session: AsyncS
     if not user:
         raise LoginException("User not found")
 
-    sign = Sign(url.url)
+    sign = Sign(url=sign.url, category_id=sign.category)
 
     # check if the sign already exists
     signs = await Sign.get_all_where(Sign.url == sign.url, session=session)