Initial commit

core/.gitignore

@@ -0,0 +1,47 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+
+# C extensions
+*.so
+
+# Distribution / packaging
+bin/
+build/
+develop-eggs/
+dist/
+eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+.tox/
+.coverage
+.cache
+nosetests.xml
+coverage.xml
+
+# Translations
+*.mo
+
+# Mr Developer
+.mr.developer.cfg
+.project
+.pydevproject
+
+# Rope
+.ropeproject
+
+# Django stuff:
+*.log
+*.pot

core/CHNAGELOG.md

@@ -0,0 +1,27 @@
+# Change Log
+
+All notable changes to core will be documented in this file.
+
+## [Unreleased] - 1401-09-03
+
+### Added
+
+- Add limiter utility
+
+### Changed
+
+- Change Config System to Yaml And DataClass
+
+## [Unreleased] - 1401-08-12
+
+💪 Powerup and isolation core module 🥇
+
+Add logger module in core
+
+### Added
+
+- Logger Modules in core
+
+### Changed
+
+- Change config file and packed in core 

core/README.md

@@ -0,0 +1,2 @@
+# HacoTech Core Server
+# Base on Fastapi 

core/app/__init__.py

@@ -0,0 +1,112 @@
+import pathlib
+from functools import lru_cache
+
+import toml
+from fastapi import FastAPI
+from fastapi.encoders import jsonable_encoder
+from fastapi.exceptions import RequestValidationError
+from fastapi.openapi.docs import get_redoc_html
+from fastapi.openapi.utils import get_openapi
+from fastapi.staticfiles import StaticFiles
+from starlette import status
+from starlette.middleware.cors import CORSMiddleware
+from starlette.responses import JSONResponse
+
+from config import conf
+from core.app.context import useAppContext
+from core.app.docs import get_spotlight_docs
+
+app = FastAPI(debug=True, root_path=conf.server.base_api_doc_url)
+
+app.state = useAppContext()
+
+# mount and map static directory
+
+for _route, _dir in conf.static_files.items():
+    app.mount(f'{conf.server.prefix_url}/{_route}',
+              StaticFiles(directory=_dir), name=_dir)
+
+
+def _parse_toml(pyproject_path):
+    pyproject_text = pyproject_path.read_text()
+    pyproject_data = toml.loads(pyproject_text)
+    info = pyproject_data['tool']['poetry']
+    setup_data = {
+        'name': info.get('name', 'No Name'),
+        'version': info.get('version', 'No version'),
+        'description': info.get('description', 'No description'),
+        'authors': info.get('authors', ['None']),
+    }
+    return setup_data
+
+
+def get_pkg_info():
+    pyproject_path = pathlib.Path('pyproject.toml')
+    return _parse_toml(pyproject_path)
+
+
+@app.exception_handler(RequestValidationError)
+async def validation_exception_handler(request, exc: RequestValidationError):
+    return JSONResponse(
+        status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+        content=jsonable_encoder({"data": exc.errors(), "message": 'Validation error', 'code': 422}),
+    )
+
+
+@app.get('/ping')
+def ping():
+    return {"message": "pong", 'api_version': 2, "production": conf.server.is_prod, **get_pkg_info()}
+
+
+def custom_openapi():
+    if app.openapi_schema:
+        return app.openapi_schema
+    info = get_pkg_info()
+    openapi_schema = get_openapi(
+        title=f'{info.get("name")} Api Documents',
+        version=info.get("version"),
+        description=info.get("description"),
+        routes=app.routes,
+    )
+    openapi_schema["info"]["x-logo"] = {
+        "url": conf.server.api_logo
+    }
+    app.openapi_schema = openapi_schema
+    return app.openapi_schema
+
+
+@app.get("/redoc", include_in_schema=False)
+async def redoc_html():
+    info = get_pkg_info()
+    return get_redoc_html(
+        openapi_url=app.openapi_url,
+        title=info.get('name') + " - ReDoc",
+    )
+
+
+@app.get("/docu", include_in_schema=False)
+async def docu():
+    info = get_pkg_info()
+    return get_spotlight_docs(
+        openapi_url=app.openapi_url,
+        title=info.get('name') + " - Docu",
+    )
+
+
+app.openapi = custom_openapi
+
+
+@lru_cache()
+def load_origins():
+    return conf.cors.origins or []
+
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=load_origins(),
+    allow_credentials=conf.cors.allow_credentials,
+    allow_methods=conf.cors.allow_methods,
+    allow_headers=conf.cors.allow_headers,
+)
+
+app.router.redirect_slashes = False

core/app/context.py

@@ -0,0 +1,13 @@
+from starlette.datastructures import State
+from starlette.requests import Request
+
+
+class AppContext(State):
+    request: Request
+
+
+app_context = AppContext()
+
+
+def useAppContext() -> AppContext:
+    return app_context

core/app/docs.py

@@ -0,0 +1,26 @@
+from starlette.responses import HTMLResponse
+
+
+def get_spotlight_docs(title, openapi_url):
+    html = f"""
+    <!doctype html>
+    <html lang="en">
+      <head>
+        <meta charset="utf-8">
+        <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+        <title>{title}</title>
+
+        <script src="https://unpkg.com/@stoplight/elements/web-components.min.js"></script>
+        <link rel="stylesheet" href="https://unpkg.com/@stoplight/elements/styles.min.css">
+      </head>
+      <body>
+
+        <elements-api
+          apiDescriptionUrl="{openapi_url}"
+            router="hash"
+        />
+
+      </body>
+    </html>
+    """
+    return HTMLResponse(html)

core/config.py

@@ -0,0 +1,129 @@
+import os
+from pathlib import Path
+from typing import Optional, List, Union
+
+import yaml
+from pydantic import BaseModel
+
+
+def build_path(path: Union[Path, str], path_is_absolute: bool = False) -> Path:
+    """
+    This function build Path instance from arguments and returns it.
+    :param path: path
+    :param path_is_absolute: if True, use path as absolute
+    :return: Path instance
+    """
+    if not path_is_absolute:
+        return Path(os.getcwd()) / path
+    if isinstance(path, str):
+        return Path(path)
+    return path
+
+
+def load_yaml(path: Union[Path, str] = None, path_is_absolute: bool = False):
+    """
+    This method loads from YAML file to properties of self instance.
+    Why doesn't load when __init__ is to make the following requirements compatible:
+    1. Access config as global
+    2. Independent on config for development or use config for unit testing when unit testing
+    """
+    if path is None:
+        path = 'config.yml'
+    built_path = build_path(path, path_is_absolute)
+    dictionary_config = yaml.full_load(built_path.read_text(encoding="UTF-8"))
+    return dictionary_config
+
+
+class Modules:
+    MIDDLEWARE: list = []
+    ROUTES: list = []
+
+
+class AuthConf:
+    AUTH_ACCESS_TOKEN_EXPIRE_MINUTES: int
+    AUTH_REFRESH_TOKEN_EXPIRE_MINUTES: int
+    AUTH_ALGORITHM: str
+    AUTH_SECRET_KEY: str
+
+
+class MongoConf:
+    MONGO_URL: str
+    MONGO_DB: str
+    AUTH_DB: str
+    PROD_MONGO_URL: str
+    PROD_MONGO_DB: str
+    PROD_AUTH_DB: str
+
+
+class LinkerConf:
+    LINKER_URL: str
+
+
+class DatabaseConfig(BaseModel):
+    url: str
+    db: str
+    version: Optional[float]
+    username: Optional[str]
+    password: Optional[str]
+
+
+class DatabaseModeConfig(BaseModel):
+    dev: Optional[DatabaseConfig]
+    production: Optional[DatabaseConfig]
+
+
+class DependencyConfig(BaseModel):
+    middleware: Optional[List[str]]
+    routes: Optional[List[str]]
+
+
+class AuthConfig(BaseModel):
+    secret_key: Optional[str]
+    algorithm: Optional[str]
+    access_token_expire_minutes: Optional[int]
+    refresh_token_expire_minutes: Optional[int]
+    url: Optional[str]
+    cache_prefix_key: Optional[str]
+    cache_ttl: Optional[int] = 360
+
+
+
+class ServerConfig(BaseModel):
+    base_api_doc_url: Optional[str]
+    is_prod: Optional[bool]
+    api_logo: Optional[str] = 'https://scm.hacoupian.net/resource/img/logo.svg'
+    prefix_url: Optional[str]
+    port: Optional[int]
+
+
+class CeleryTask(BaseModel):
+    broker: Optional[str]
+    backend: Optional[str]
+    autodiscover_tasks: Optional[List[str]]
+
+
+class CorsConfig(BaseModel):
+    origins: Optional[List[str]]
+    allow_headers: Optional[List[str]] = ['*']
+    allow_credentials: Optional[bool] = True
+    allow_methods: Optional[List[str]] = ['*']
+
+
+class CoreConfig(BaseModel):
+    cors: Optional[CorsConfig]
+    server: Optional[ServerConfig]
+    auth: Optional[AuthConfig]
+    dependency: Optional[DependencyConfig]
+    database: dict[str, Union[DatabaseConfig, DatabaseModeConfig]]
+    static_files: dict[str, str] = dict()
+    celery: Optional[CeleryTask]
+
+    class Config:
+        arbitrary_types_allowed = False
+
+    # def load_modules_config(self):
+    #     #  import dynamic routes in the modules
+    #     for i in self.dependency.routes:
+    #         if os.path.exists(Path(f'{i}/.env')):
+    #             values = dotenv_values(f'{i}/.env')
+    #             self.extra_conf = dict(self.extra_conf, **dict(values))

core/exceptions/__init__.py

@@ -0,0 +1 @@
+from .model import *

core/exceptions/errors.py

@@ -0,0 +1,81 @@
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class BaseError(BaseModel):
+    message: str = Field(..., description="Error message or description")
+
+
+class BaseIdentifiedError(BaseError):
+    identifier: str = Field(..., description="Unique identifier which this error references to")
+
+
+class NotFoundError(BaseIdentifiedError):
+    """The entity does not exist"""
+    pass
+
+
+class AlreadyExistsError(BaseIdentifiedError):
+    """An entity being created already exists"""
+    pass
+
+
+class ModifiedCountException(Exception):
+    message: str = Field(..., description="Error message or description")
+
+    def __init__(self, identifier: any, message: str = 'Data not update', **data: Any):
+        super().__init__()
+        if identifier:
+            self.message = f'data with id {identifier} not update'
+        else:
+            self.message = message
+
+
+class ValidationException(Exception):
+    message: str = Field(..., description="Error message or description")
+
+    def __init__(self, message: str = 'validation exception', **data: Any):
+        super().__init__()
+        self.message = f'{message} with {data}' if data else message
+
+
+class RecordNotFoundException(Exception):
+    message: str = Field(..., description="Error message or description")
+
+    def __init__(self, identifier: any = None, message: str = 'record not found', **data: Any):
+        super().__init__()
+        if identifier:
+            self.message = f'data with id {identifier} not found'
+        else:
+            self.message = message
+
+
+class SoftDeletedException(Exception):
+    message: str = Field(..., description="Error message or description")
+
+    def __init__(self, identifier: any, message: str = 'Data not soft deleted', **data: Any):
+        super().__init__()
+        if identifier:
+            self.message = f'data with id {identifier} not soft deleted'
+        else:
+            self.message = message
+
+
+class NotFoundException(Exception):
+    message: str = Field(..., description="Error message or description")
+
+    def __init__(self, identifier: any, message: str = 'Data not found', **data: Any):
+        super().__init__()
+        if identifier:
+            self.message = f'data with id {identifier} not found'
+        else:
+            self.message = message
+
+
+class BadRequestException(Exception):
+    message: str = Field(..., description="Error message or description")
+
+    def __init__(self, message: str = 'Data bad data', **data: Any):
+        super().__init__()
+        self.message = message

core/exceptions/model.py

@@ -0,0 +1,80 @@
+from typing import Type
+
+from fastapi import status as statuscode
+from fastapi.responses import JSONResponse
+
+# # Package # #
+from .errors import *
+
+
+class BaseAPIException(Exception):
+    """Base error for custom API exceptions"""
+    message = "Generic error"
+    code = statuscode.HTTP_500_INTERNAL_SERVER_ERROR
+    model = BaseError
+
+    def __init__(self, **kwargs):
+        kwargs.setdefault("message", self.message)
+        self.message = kwargs["message"]
+        self.data = self.model(**kwargs)
+
+    def __str__(self):
+        return self.message
+
+    def response(self):
+        return JSONResponse(
+            content=self.data.dict(),
+            status_code=self.code
+        )
+
+    @classmethod
+    def response_model(cls):
+        return {cls.code: {"model": cls.model}}
+
+
+class BaseIdentifiedException(BaseAPIException):
+    """Base error for exceptions related with entities, uniquely identified"""
+    message = "Entity error"
+    code = statuscode.HTTP_500_INTERNAL_SERVER_ERROR
+    model = BaseIdentifiedError
+
+    def __init__(self, identifier, **kwargs):
+        super().__init__(identifier=identifier, **kwargs)
+
+
+class NotFoundException(BaseIdentifiedException):
+    """Base error for exceptions raised because an entity does not exist"""
+    message = "The entity does not exist"
+    code = statuscode.HTTP_404_NOT_FOUND
+    model = NotFoundError
+
+
+class AlreadyExistsException(BaseIdentifiedException):
+    """Base error for exceptions raised because an entity already exists"""
+    message = "The entity already exists"
+    code = statuscode.HTTP_409_CONFLICT
+    model = AlreadyExistsError
+
+
+class ColorConstNotFoundException(NotFoundException):
+    """Error raised when a person does not exist"""
+    message = "The const does not exist"
+
+
+class ConstNotFoundException(NotFoundException):
+    """Error raised when a person does not exist"""
+    message = "The const does not exist"
+
+
+class PersonAlreadyExistsException(AlreadyExistsException):
+    """Error raised when a person already exists"""
+    message = "The const already exists"
+
+
+def get_exception_responses(*args: Type[BaseAPIException]) -> dict:
+    """Given BaseAPIException classes, return a dict of responses used on FastAPI endpoint definition, with the format:
+    {statuscode: schema, statuscode: schema, ...}"""
+    responses = dict()
+    for cls in args:
+        responses.update(cls.response_model())
+    return responses

core/middleware/__init__.py

@@ -0,0 +1,3 @@
+from .auth import *
+from .logger_middleware import *
+from .prod_middleware import *

core/middleware/auth.py

@@ -0,0 +1,17 @@
+from fastapi import Depends
+from starlette.authentication import AuthenticationBackend, AuthCredentials
+from starlette.middleware.authentication import AuthenticationMiddleware
+
+from core.utils.auth import get_current_user
+
+
+class OAuth2Backend(AuthenticationBackend):
+    async def authenticate(self, request, user=Depends(get_current_user)):
+        return AuthCredentials(["authenticated"]), user
+
+
+def setup_auth_middleware(app):
+    app.add_middleware(
+        AuthenticationMiddleware,
+        backend=OAuth2Backend(),
+    )

core/middleware/logger_middleware.py

@@ -0,0 +1,43 @@
+import time
+
+from starlette.requests import Request
+
+from core.app import app
+from core.modules import logger
+from core.utils.auth import decode_token
+
+
+@app.middleware("http")
+async def logger_middleware(request: Request, call_next):
+    start_time = time.time()
+    app.state.request = request
+    response = await call_next(request)
+    try:
+        user = request.state.user if hasattr(request.state, "user") else None
+        token = (
+            decode_token(request.headers.get("Authorization"))
+            if request.headers.get("Authorization")
+            else None
+        )
+        process_time = time.time() - start_time
+        # logger.emit_access(
+        #     {
+        #         "ip": request.client.host,
+        #         "method": request.method,
+        #         "path": request.url.path,
+        #         "p_time": float("{0:.3f}".format(process_time)),
+        #         "status": response.status_code,
+        #         "uid": token.get("sub") if token else None,
+        #         "scopes": token.get("scopes") if token else None,
+        #         "name": user.name if user else None,
+        #         "level": response.status_code >= 400 and "error" or "info",
+        #         "user_agent": httpagentparser.simple_detect(
+        #             request.headers.get("User-Agent")
+        #         ),
+        #         "referer": request.headers.get("Referer"),
+        #     }
+        # )
+    except Exception as e:
+        logger.emit({'error': str(e)}, logger.Types.Error)
+    finally:
+        return response

core/middleware/prod_middleware.py

@@ -0,0 +1,20 @@
+from fastapi.requests import Request
+
+from config import conf
+from core.app import app
+
+
+@app.middleware("http")
+async def prod_middleware(request: Request, call_next):
+    if (
+            request.url.hostname == "scm.haco.dc"
+            or request.url.hostname == "scm.hacoupian.net"
+            or request.url.hostname == "rc.hacoupian.net"
+    ):
+        conf.server.is_prod = True
+    else:
+        conf.server.is_prod = False
+    response = await call_next(request)
+    response.headers["is-host"] = request.url.hostname
+    response.headers["is-prod"] = str(conf.server.is_prod)
+    return response

core/models/base.py

@@ -0,0 +1,167 @@
+import uuid
+from datetime import datetime
+from typing import Optional, List
+
+import pydantic
+from pydantic import Field, create_model
+from pydantic.fields import ModelField, FieldInfo
+
+from core.service.mongo_db import use_collection
+from core.utils import get_time, get_uuid
+
+MORPH_INPUT_UPDATE = ['updated_at', 'created_at', 'id', 'deleted_at', 'owner_id']
+MORPH_INPUT_CREATE = ['updated_at', 'created_at', 'id', 'deleted_at', 'owner_id']
+
+
+class BaseModel(pydantic.BaseModel):
+    """All data models inherit from this class"""
+    owner_id: Optional[uuid.UUID] = Field(description='ایجاد کننده رکورد')
+    created_at: Optional[datetime] = Field(description='تاریخ ایجاد رکورد')
+    updated_at: Optional[datetime] = Field(description='تاریخ آخرین بروزرسانی رکورد')
+
+    class At:
+        update: List[str]
+
+    @pydantic.root_validator(pre=True)
+    def _min_properties(cls, data):
+        """At least one property is required"""
+        if not data:
+            raise ValueError("At least one property is required")
+        return data
+
+    def dict(self, include_nulls=False, **kwargs):
+        """Override the super dict method by removing null keys from the dict, unless include_nulls=True"""
+        kwargs["exclude_none"] = not include_nulls
+        result = super().dict(**kwargs)
+        return result
+
+    class Config:
+        extra = pydantic.Extra.ignore
+        anystr_strip_whitespace = True  # strip whitespaces from strings
+        underscore_attrs_are_private = False
+        json_encoders = {
+            uuid.UUID: lambda v: str(v),
+        }
+
+    @classmethod
+    def to_partial(cls):
+        return get_partial_model(cls)
+
+    def mongo_dict(self, **kwargs):
+        result = self.dict(**kwargs)
+        result['_id'] = result.get('id', None)
+        result.pop('id', None)
+        return result
+
+    def dict_without_id(self, **kwargs):
+        result = self.dict(**kwargs)
+        result.pop('id', None)
+        return result
+
+    def cu_at(self):
+        self.c_at()
+        self.u_at()
+        return self
+
+    def c_at(self):
+        self.created_at = get_time()
+        return self
+
+    def u_at(self):
+        self.updated_at = get_time()
+        return self
+
+    def init_id(self):
+        self.id = get_uuid()
+        return self
+
+    def init_owner(self, is_array=False, key="owner_id"):
+        from core.utils.app_context import use_user
+        if is_array:
+            setattr(self, key, [use_user().id if use_user() else None])
+        else:
+            setattr(self, key, use_user().id if use_user() else None)
+        return self
+
+    async def save(self):
+        if self.DB.collection is None:
+            raise ValueError("collection is not set")
+        record = await use_collection(self.DB.collection).insert_one(self.u_at().mongo_dict())
+        if not record.inserted_id:
+            raise ValueError("Failed to save record")
+        return record.inserted_id
+
+    @classmethod
+    def at(cls, op: str):
+        __cols = getattr(cls.At, op)
+        if __cols:
+            all_fields = dict()
+            for key, item in cls.__fields__.items():
+                if key not in __cols:
+                    if not item.required:
+                        # TODO: must be correct documents
+                        all_fields.setdefault(key, (item.type_, FieldInfo(
+                            default=item.default,
+                            description=item.field_info.description or item.__doc__,
+                            default_factory=item.field_info.default_factory,
+                            alias=item.field_info.alias,
+                            alias_priority=item.field_info.alias_priority,
+                            title=item.field_info.title,
+                            exclude=item.field_info.exclude,
+                            include=item.field_info.include,
+                            const=item.field_info.const,
+                            gt=item.field_info.gt,
+                            ge=item.field_info.ge,
+                            lt=item.field_info.lt,
+                            le=item.field_info.le,
+                            multiple_of=item.field_info.multiple_of,
+                            allow_inf_nan=item.field_info.allow_inf_nan,
+                            max_digits=item.field_info.max_digits,
+                            decimal_places=item.field_info.decimal_places,
+                            min_items=item.field_info.min_items,
+                            max_items=item.field_info.max_items,
+                            unique_items=item.field_info.unique_items,
+                            min_length=item.field_info.min_length,
+                            max_length=item.field_info.max_length,
+                            allow_mutation=item.field_info.allow_mutation,
+                            repr=item.field_info.repr,
+                            regex=item.field_info.regex,
+                            discriminator=item.field_info.discriminator,
+                            extra=item.field_info.extra,
+                        )))
+                    else:
+                        all_fields.setdefault(key, (item.type_, ...))
+            DynamicModel = create_model(cls.__name__, **all_fields)
+            return DynamicModel
+        return cls
+
+
+class BaseSoftDelete(pydantic.BaseModel):
+    deleted_at: Optional[datetime] = Field(title='زمان حذف رکورد',
+                                           description='زمانی که این فیلد در داده وجود داشته باشد نشان دهنده این است که این رکورد حذف نرم شده است.')
+
+
+class BaseCUAt(pydantic.BaseModel):
+    created_at: Optional[datetime]
+    updated_at: Optional[datetime]
+
+
+DELETED_AT_QUERY = {'deleted_at': {'$exists': False}}
+
+ORDER_BY_CREATED_AT = {'key_or_list': 'created_at', 'direction': -1}
+
+
+def get_partial_model(model: BaseModel) -> BaseModel:
+    """
+    Return a model similar to the original, but where all fields are Optional.
+
+    Note: this minimal implementation means that many Pydantic features will be discarded
+    (such as alias). This is not by design and could stand to improve.
+    """
+    new_fields = {name: (Optional[model.__annotations__.get(name)], model.__fields__[
+        name].default or None) for name in model.__fields__}
+
+    return pydantic.create_model(
+        "PartialModel",
+        **new_fields,
+        __base__=BaseModel)

core/models/object_id.py

@@ -0,0 +1,17 @@
+from bson import ObjectId
+
+
+class PyObjectId(ObjectId):
+    @classmethod
+    def __get_validators__(cls):
+        yield cls.validate
+
+    @classmethod
+    def validate(cls, v):
+        if not ObjectId.is_valid(v):
+            raise ValueError("Invalid objectid")
+        return ObjectId(v)
+
+    @classmethod
+    def __modify_schema__(cls, field_schema):
+        field_schema.update(type="string")

core/models/table_engine.py

@@ -0,0 +1,7 @@
+from pydantic import BaseModel
+
+
+class TableEngineInput(BaseModel):
+    """
+    TableEngineInput class
+    """

core/models/unit.py

@@ -0,0 +1,8 @@
+import uuid
+
+from core.models.base import BaseModel
+
+
+class UnitAccess(BaseModel):
+    access: int
+    unit_id: uuid.UUID

core/modules/logger/__init__.py

@@ -0,0 +1,57 @@
+from enum import Enum
+from typing import Union
+
+from fluent import event
+from fluent import sender
+
+
+class Types(Enum):
+    Default = 'default'
+    EditData = 'edit_data'
+    CreateData = 'create_data'
+    DeleteData = 'delete_data'
+    Auth = 'authentication'
+    System = 'system'
+    Error = 'error'
+    Upgrade = 'upgrade'
+    Access = 'access'
+
+
+def emit(payload: dict, tag: Union[Types, str] = Types.Default) -> None:
+    sender.setup('mongo', host='localhost', port=24224)
+    # try:
+    #     owner = useUser().id
+    #     payload['owner'] = str(owner)
+    # except BaseException:
+    #     payload['owner'] = 'anonymous'
+    # finally:
+    val = tag.value if isinstance(tag, Types) else tag
+    event.Event(val, payload)
+
+
+def update(dest: str, payload: dict) -> None:
+    emit({'dest': dest, **payload}, Types.EditData)
+
+
+def insert(dest: str, payload: dict) -> None:
+    emit({'dest': dest, **payload}, Types.CreateData)
+
+
+def delete(dest: str, payload: dict) -> None:
+    emit({'dest': dest, **payload}, Types.DeleteData)
+
+
+def access(payload: dict) -> None:
+    emit(payload, Types.Access)
+
+
+def auth(payload: dict) -> None:
+    emit(payload, Types.Auth)
+
+
+def system(payload: dict) -> None:
+    emit(payload, Types.System)
+
+
+def upgrade(payload: dict) -> None:
+    emit(payload, Types.Upgrade)

core/repositories/base.py

@@ -0,0 +1,47 @@
+from typing import Generic, TypeVar, Any
+from uuid import UUID
+
+from pydantic.generics import GenericModel
+
+from core.service.mongo_db import use_collection
+from core.utils.crudify import mongo_inserter, mongo_soft_deleter, mongo_updater, mongo_find_one
+from core.utils.table_engine import TableEngineRequestType, TableEngineExport
+from core.utils.te import NewTableEngine
+
+T = TypeVar('T')
+
+
+class BaseRepository(GenericModel, Generic[T]):
+    COLLECTION: str = 'asa'
+    FETCH_COLS_SEARCH = ['v_id']
+    MODEL: T = None
+
+    def __init__(self, collection: str, fetch_cols_search=None, model: T = None, **data: Any):
+        super().__init__(**data)
+        if fetch_cols_search is None:
+            fetch_cols_search = ['v_id']
+        self.COLLECTION = collection
+        self.MODEL = model
+        self.FETCH_COLS_SEARCH = fetch_cols_search
+
+    async def get(self, _id: UUID) -> T:
+        return await mongo_find_one(_id, self.COLLECTION)
+
+    async def fetch(self, _input: TableEngineRequestType) -> TableEngineExport[T]:
+        return await NewTableEngine.mongo(
+            query_class=use_collection(self.COLLECTION),
+            soft_deleted=True,
+            model=self.MODEL,
+            search_cols=self.FETCH_COLS_SEARCH,
+            inputs=_input.te) \
+            .to_dict()
+
+    async def create(self, create: T) -> T:
+        return await mongo_inserter(create, self.COLLECTION, getter=self.get)
+
+    async def update(self, _id: UUID, update: T, extra_query: dict = None):
+        return await mongo_updater(update, _id, self.COLLECTION, getter=self.get,
+                                   extra_query=extra_query)
+
+    async def delete(self, _id: UUID):
+        return await mongo_soft_deleter(_id, self.COLLECTION)

core/service/limiter.py

@@ -0,0 +1,15 @@
+from slowapi import Limiter, _rate_limit_exceeded_handler
+from slowapi.errors import RateLimitExceeded
+from starlette.requests import Request
+
+from config import conf
+from core.app import app
+
+
+def get_unique_identification(request: Request) -> str:
+    return request.client.host or "127.0.0.1"
+
+
+limiter = Limiter(key_func=get_unique_identification, storage_uri=conf.database.get('limiter_redis').url)
+app.state.limiter = limiter
+app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler)

core/service/mongo_db.py

@@ -0,0 +1,14 @@
+import motor.motor_asyncio
+from pymongo.collection import Collection
+
+from core.utils.mongo import get_mongo_url, get_mongo_db
+
+
+def use_client(connection_name: str = None):
+    return motor.motor_asyncio.AsyncIOMotorClient(get_mongo_url(connection_name))
+
+
+def use_collection(collection: str, db: str = None, connection_name: str = None) -> Collection:
+    if not db:
+        db = get_mongo_db(connection_name)
+    return use_client(connection_name)[db][collection]

core/service/redis.py

@@ -0,0 +1,34 @@
+import json
+
+import aioredis
+from aioredis import Redis
+from aioredis.client import ExpiryT
+
+from core.config import DatabaseConfig
+
+
+def redis_db(db: DatabaseConfig) -> Redis:
+    # Redis client bound to pool of connections (auto-reconnecting).
+    redis = aioredis.from_url(
+        db.url,
+        encoding="utf-8",
+        decode_responses=True,
+        db=int(db.db) or 0
+    )
+    return redis
+
+
+async def redis_set(key: str, value: any, db: DatabaseConfig, ex=ExpiryT):
+    redis_c = redis_db(db)
+    await redis_c.set(key, json.dumps(value), ex=ex)
+    await redis_c.close()
+    return True
+
+
+async def redis_get(key: str, db: DatabaseConfig):
+    redis_c = redis_db(db)
+    _return = await redis_c.get(key)
+    await redis_c.close()
+    if not _return or isinstance(_return, (int, float, complex)):
+        return _return
+    return json.loads(_return)

core/urls/__init__.py

@@ -0,0 +1 @@
+from .sumery import *

core/urls/sumery.py

@@ -0,0 +1,38 @@
+from typing import List
+
+from fastapi import Request
+from pydantic import create_model
+
+from config import conf
+from core.app import app
+from core.utils.router import generate_router, safe_route
+from core.utils.sumery import sumery_executor, sumery_default_getter
+
+router = generate_router("com/sumery", auth_dependencies=True, version='v2')
+
+_tag_base = ['sumery']
+_pre = conf.server.prefix_url.replace('v1', 'v2')
+
+
+@router.get('/{_object}', response_model=List, tags=_tag_base)
+async def smart_query(q: str, _object: str, request: Request):
+    params = request.query_params
+    extra_filter = {}
+    for key, value in params.items():
+        if key.startswith('e_'):
+            extra_filter.setdefault(key[2:], value)
+
+    return await safe_route('get', sumery_executor, _object, q, extra_filter)
+
+
+@router.post('/default/{_object}', response_model=List, tags=_tag_base)
+async def smart_query(_object: str, _input: create_model('InputDefaultSumery', ids=[]), request: Request):
+    params = request.query_params
+    extra_filter = {}
+    for key, value in params.items():
+        if key.startswith('e_'):
+            extra_filter.setdefault(key[2:], value)
+    return await safe_route('get', sumery_default_getter, _object, _input.ids, extra_filter)
+
+
+app.include_router(router)

core/utils/__init__.py

@@ -0,0 +1,8 @@
+import os
+
+from .utils import *
+from .response import *
+
+
+def get_path(__file__=__file__):
+    return os.path.abspath(os.path.dirname(__file__)) + '/'

core/utils/api.py

@@ -0,0 +1,5 @@
+from config import conf
+
+
+def get_namespace_api(prefix):
+    return f"{conf.server.prefix_url}/{prefix}"

core/utils/app_context.py

@@ -0,0 +1,21 @@
+from starlette.requests import Request
+
+from core.app import app
+from core.utils.auth import UserModel
+
+
+def use_request() -> Request:
+    try:
+        return app.state.request.state
+    except AttributeError:
+        raise AttributeError("No request object found in app.state")
+
+
+def use_user(user=None) -> UserModel:
+    try:
+        if user is None:
+            return use_request().user
+        else:
+            app.state.request.state.user = user
+    except AttributeError:
+        raise AttributeError("No user object found in app.state")

core/utils/auth/__init__.py

@@ -0,0 +1,78 @@
+import httpx
+from fastapi import HTTPException, Depends, Security
+from fastapi.security import SecurityScopes
+from httpx import ConnectError
+from jose import jwt
+from starlette.requests import Request
+
+from config import conf
+from .model import UserModel, ValidateTokenInput
+from .schema import oauth2
+from ..app_context import use_user
+from ...service.redis import redis_set, redis_get
+
+
+def decode_token(token: str):
+    return jwt.decode(
+        token.split(' ').pop(), conf.auth.secret_key, algorithms=[conf.auth.algorithm])
+
+
+async def validate_token(_input: ValidateTokenInput):
+    try:
+        _cache_key = _input.token.split('.')[-1]
+        _cache_auth = await redis_get(f'{conf.auth.cache_prefix_key}{_cache_key}', conf.database.get('auth_redis'))
+        if not _cache_auth:
+            client = httpx.AsyncClient()
+            headers = {'Content-Type': 'application/json', }
+
+            result = await client.request('POST', conf.auth.url, json=_input.dict(), headers=headers)
+            export = result.text
+            if 'application/json' in result.headers['content-type']:
+                export = result.json()
+
+            if result.status_code > 299:
+                raise HTTPException(status_code=result.status_code, detail=export)
+
+            if 'v2' in conf.auth.url:
+                export = export.get('data')
+
+            await redis_set(f'{conf.auth.cache_prefix_key}{_cache_key}', export, conf.database.get('auth_redis'),
+                            ex=conf.auth.cache_ttl)
+
+            return UserModel(**export)
+        else:
+            return UserModel(**_cache_auth)
+
+    except ConnectError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    except HTTPException as e:
+        raise e
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+
+async def get_current_user(security_scopes: SecurityScopes, token: str = Depends(oauth2)):
+    if security_scopes.scopes:
+        authenticate_value = f'Bearer scope="{security_scopes.scope_str}"'
+    else:
+        authenticate_value = f"Bearer"
+    credentials_exception = HTTPException(
+        status_code=401,
+        detail="Could not validate credentials",
+        headers={"WWW-Authenticate": authenticate_value},
+    )
+    user = await validate_token(ValidateTokenInput(token=token, scopes=security_scopes.scopes))
+    if user is None:
+        raise credentials_exception
+    user.scopes = security_scopes.scopes if isinstance(
+        security_scopes.scopes, list) else security_scopes.scopes
+    return user
+
+
+async def get_current_active_user(request: Request,
+                                  current_user: UserModel = Security(get_current_user, scopes=["account"]), ):
+    if current_user.disabled:
+        raise HTTPException(status_code=400, detail="Inactive user")
+    request.state.user = UserModel(**current_user.mongo_dict())
+    use_user(UserModel(**current_user.mongo_dict()))
+    return current_user

core/utils/auth/model.py

@@ -0,0 +1,29 @@
+import uuid
+from typing import Optional, Union, List
+
+import pydantic
+from pydantic import Field
+
+from core.models.base import BaseModel
+
+
+class UserModel(BaseModel):
+    id: Optional[Union[uuid.UUID, str]] = Field(alias='_id')
+    username: Optional[str]
+    mobile: Optional[str]
+    name: Optional[str]
+    first_name: Optional[str]
+    last_name: Optional[str]
+    is_su: Optional[bool]
+    unit: Optional[str]
+    avatar: Optional[str]
+    desc: Optional[str]
+    scopes: Optional[List[str]]
+    home_path: Optional[str]
+    roles: Optional[List[uuid.UUID]]
+    disabled: Optional[bool]
+
+
+class ValidateTokenInput(pydantic.BaseModel):
+    token: str
+    scopes: Optional[List[str]] = ['account']

core/utils/auth/schema.py

@@ -0,0 +1,9 @@
+from fastapi.security import OAuth2PasswordBearer
+
+oauth2 = OAuth2PasswordBearer(
+    tokenUrl="token",
+    scopes={
+        "account": "Read information about the current user.",
+        "scm": "Scm Modules",
+    },
+)

core/utils/can.py

@@ -0,0 +1,18 @@
+from functools import wraps
+from typing import List, Union
+
+from core.utils.app_context import use_user
+
+
+def can(permissions: Union[str, List[str]]):
+    def can_inner(func):
+        @wraps(func)
+        async def wrapper(*args, **kwargs):
+            print(permissions)
+            user = use_user()
+            # raise HTTPException(status_code=403, detail='forbidden request')
+            return await func(*args, **kwargs)
+
+        return wrapper
+
+    return can_inner

core/utils/crudify.py

@@ -0,0 +1,118 @@
+from dataclasses import dataclass
+from uuid import UUID
+
+from core.exceptions import ModifiedCountException, RecordNotFoundException
+from core.models.base import BaseModel, DELETED_AT_QUERY
+from core.service.mongo_db import use_collection
+from core.utils import get_time, get_v_id
+
+
+@dataclass
+class ConfigMongoWriter:
+    created_at: bool = True
+    updated_at: bool = True
+    has_owner: bool = True
+    init_id: bool = True
+    init_v_id: bool = False
+
+
+async def mongo_inserter(model: BaseModel, collection: str, options: ConfigMongoWriter = ConfigMongoWriter(),
+                         getter=None, db=None, v_id_prefix=None, client=None) -> dict | BaseModel:
+    """
+    Inserts data into collection
+    """
+    try:
+        if options.init_id:
+            model.init_id()
+        if options.has_owner:
+            model.init_owner()
+        if options.created_at and options.updated_at:
+            model.cu_at()
+        if options.created_at:
+            model.c_at()
+        if options.updated_at:
+            model.u_at()
+        if options.init_v_id:
+            model.v_id = get_v_id(v_id_prefix or 'uk', source_id=model.id)
+
+        document = model.mongo_dict()
+        if client:
+            result = await client(collection).insert_one(document)
+        else:
+            result = await use_collection(collection, db).insert_one(document)
+        assert result.acknowledged
+        if getter:
+            return await getter(result.inserted_id)
+        return result
+    except AssertionError as e:
+        raise AssertionError(f'{collection} insert error')
+    except Exception as e:
+        raise e
+
+
+async def mongo_soft_deleter(query: dict | UUID, collection: str, db=None, client=None):
+    """
+    Updates data in collection
+    """
+    # doc = {"$set": {'deleted_at': get_time()}, }
+    _query = query
+    if isinstance(query, UUID):
+        _query = {'_id': query}
+    __trash_collection = f'{collection}--trash'
+    __driver = client or use_collection
+    __current_record = await __driver(collection, db=db).find_one(_query)
+    if not __current_record:
+        raise RecordNotFoundException(identifier=str(query))
+    __current_record['deleted_at'] = get_time()
+    __driver(__trash_collection, db=db).insert_one(__current_record)
+    __driver(collection, db=db).delete_one(_query)
+    return True
+
+
+async def mongo_updater(model: BaseModel,
+                        query: dict | UUID,
+                        collection: str,
+                        options: ConfigMongoWriter = ConfigMongoWriter(created_at=False, has_owner=False),
+                        getter=None,
+                        db=None,
+                        extra_query: dict = None,
+                        client=None,
+                        ):
+    """
+    Updates data in collection
+    """
+    _query = query
+    if isinstance(query, UUID):
+        _query = {'_id': query}
+    if options.has_owner:
+        model.init_owner()
+    if options.updated_at:
+        model.u_at()
+    if client:
+        result = await client(collection).update_one(_query, {"$set": model.dict(),
+                                                              **(extra_query if extra_query else {})})
+    else:
+        result = await use_collection(collection, db).update_one(_query, {"$set": model.dict(),
+                                                                          **(extra_query if extra_query else {})})
+    if not result.modified_count:
+        raise ModifiedCountException(query)
+    if getter:
+        return await getter(_query)
+    return result
+
+
+async def mongo_find_one(query: dict | UUID, collection: str, db=None, client=None, project=None) -> dict:
+    """
+     Ge
+     """
+    _query = query
+    if isinstance(query, UUID):
+        _query = {'_id': query}
+    _query = dict(DELETED_AT_QUERY, **_query)
+    if client:
+        result = await client(collection).find_one(_query, project)
+    else:
+        result = await use_collection(collection, db).find_one(_query, project)
+    if not result:
+        raise RecordNotFoundException(identifier=query.__str__())
+    return result

core/utils/json.py

@@ -0,0 +1,42 @@
+import json
+from datetime import datetime
+from typing import Optional, Any, Union
+from uuid import UUID
+
+#
+# def uuid_encoder():
+#     print(obj)
+#     if isinstance(obj, UUID):
+#         # if the obj is uuid, we simply return the value of uuid
+#         return obj.hex
+#     return json.JSONEncoder.default(self, obj)
+#
+import jsonpatch
+from pydantic import Field
+
+from core.models.base import BaseModel
+
+
+class DataModelPatch(BaseModel):
+    id: Optional[Union[UUID, str]] = Field(alias='_id')
+    patch: Optional[Any]
+    ref_id: UUID
+    collection: str
+    owner: UUID
+    created_at: Optional[datetime]
+
+    class DB:
+        collection: str = 'revisions'
+
+
+async def set_revision(collection: str, old: str | dict, new: str | dict, ref_id: UUID, user_id: UUID):
+    old = json.dumps(old, default=str) if isinstance(old, dict) else old
+    new = json.dumps(new, default=str) if isinstance(new, dict) else new
+    patch = jsonpatch.JsonPatch.from_diff(
+        json.loads(new), json.loads(old)
+    )
+    await DataModelPatch(
+        patch=json.loads(patch.to_string()),
+        ref_id=str(ref_id),
+        collection=collection,
+        owner=str(user_id)).init_id().c_at().save()

core/utils/mongo.py

@@ -0,0 +1,47 @@
+from typing import List, Dict, Any
+
+from pymongo.collection import Collection
+
+from config import conf
+from core.exceptions import NotFoundException
+from core.utils import get_time
+
+
+def get_mongo_url(connection_name:str = None):
+    return conf.server.is_prod and conf.database.get(connection_name or 'mongo').production.url or conf.database.get(
+        connection_name or 'mongo').dev.url
+
+
+def get_mongo_db(connection_name:str = None):
+    return conf.server.is_prod and conf.database.get(connection_name or 'mongo').production.db or conf.database.get(
+        connection_name or 'mongo').dev.db
+
+
+def nested_join(collection: str, from_field, to_field,
+                alias, match: dict) -> List[Dict[str, Any]]:
+    """
+        Join a list of nested dictionaries into a single dictionary.
+    """
+    return [
+        {
+            "$lookup": {
+                "from": collection,
+                "localField": from_field,
+                "foreignField": to_field,
+                "as": alias
+            },
+        },
+        {
+            "$match": match if match else {}
+        }
+    ]
+
+
+async def use_soft_delete(collection: Collection, find_query: dict) -> bool:
+    """
+        Soft delete a document by setting a field to true.
+    """
+    result = await collection.update_one(find_query, {"$set": {'deleted_at': get_time()}, })
+    if not result.modified_count:
+        raise NotFoundException(f"No document found with query: {find_query}")
+    return True

core/utils/number.py

@@ -0,0 +1,12 @@
+import re
+
+E2P_map = {'1': '۱', '2': '۲', '3': '۳', '4': '۴', '5': '۵', '6': '۶', '7': '۷', '8': '۸', '9': '۹', '0': '۰'}
+
+E2P_map = {'1': '۱', '2': '۲', '3': '۳', '4': '۴', '5': '۵', '6': '۶', '7': '۷', '8': '۸', '9': '۹', '0': '۰'}
+
+
+# convert persian number to english number
+def convert_number_to_persian(str_in: str):
+    a = map(lambda ch: E2P_map[ch] if ch in E2P_map else ch, str_in)
+    return ''.join(list(a))
+

core/utils/response.py

@@ -0,0 +1,163 @@
+from fastapi import HTTPException
+from fastapi.encoders import jsonable_encoder
+from pydantic import ValidationError
+from starlette.responses import JSONResponse
+
+from core.exceptions import SoftDeletedException, RecordNotFoundException, ModifiedCountException, ValidationException
+
+
+class ResponseModel:
+    def __init__(self, data, code=200, message="ok"):
+        self.data = data
+        self.code = code
+        self.message = message
+
+    def __str__(self):
+        return str(self.data)
+
+    def __iter__(self):
+        return {
+            "data": self.data,
+            "code": self.code,
+            "message": self.message,
+        }
+
+
+class ResponseBadRequest(ResponseModel):
+    def __init__(self, data, message="Bad Request"):
+        super().__init__(data, 400, message)
+
+
+class ResponseValidationRequest(ResponseModel):
+    def __init__(self, data, message="Bad Request"):
+        super().__init__(data, 422, message)
+
+
+class ResponseWriteErrorRequest(ResponseModel):
+    def __init__(self, data, message="Request write error"):
+        super().__init__(data, 400, message)
+
+
+class ResponseNotFound(ResponseModel):
+    def __init__(self, data, message="Not Found"):
+        super().__init__(data, 404, message)
+
+
+class ResponseUnauthorized(ResponseModel):
+    def __init__(self, data, message="Unauthorized"):
+        super().__init__(data, 401, message)
+
+
+class ResponseInternalServerError(ResponseModel):
+    def __init__(self, data, message="Internal Server Error"):
+        super().__init__(data, 500, message)
+
+
+class ResponseOk(ResponseModel):
+    def __init__(self, data, code=200, message="ok"):
+        super().__init__(data, code, message)
+
+
+class ResponseOkUpdate(ResponseOk):
+    def __init__(self, data, message="update was successful"):
+        super().__init__(data, 200, message)
+
+
+class ResponseOkCreate(ResponseOk):
+    def __init__(self, data, message="create was successful"):
+        super().__init__(data, 201, message)
+
+
+class Res:
+
+    @staticmethod
+    def json(object, status_code):
+        return JSONResponse(content=jsonable_encoder(object), status_code=status_code)
+
+    @staticmethod
+    def create(data, message="create was successful"):
+        return Res.json(ResponseOkCreate(data, message), 201)
+
+    @staticmethod
+    def fetch(data, message="fetch was successful"):
+        return Res.json(ResponseOk(data, 200, message), 200)
+
+    @staticmethod
+    def update(data, message="update was successful"):
+        return Res.json(ResponseOkUpdate(data, message), 200)
+
+    @staticmethod
+    def not_found(data, message="Not Found"):
+        return Res.json(ResponseNotFound(data, message), 404)
+
+    @staticmethod
+    def unauthorized(data, message="Unauthorized"):
+        return Res.json(ResponseUnauthorized(data, message), 401)
+
+    @staticmethod
+    def server_error(data, message="Internal Server Error"):
+        return Res.json(ResponseInternalServerError(data, message), 500)
+
+    @staticmethod
+    def bad_request(data, message="Bad Request"):
+        return Res.json(ResponseBadRequest(data, message), 400)
+
+    @staticmethod
+    def validation_error(data, message="data validation error"):
+        return Res.json(ResponseValidationRequest(data, message), 422)
+
+    @staticmethod
+    def write_error(data, message="Request write error"):
+        return Res.json(ResponseWriteErrorRequest(data, message), 400)
+
+    @staticmethod
+    def delete_error(data, message="Request delete error"):
+        return Res.json(ResponseWriteErrorRequest(data, message), 400)
+
+
+def to_http_res(res, _type, message=None):
+    if _type == "create":
+        return Res.create(res, "create was successful")
+    elif _type == "fetch":
+        return Res.fetch(res, "fetch was successful")
+    elif _type == "update":
+        return Res.update(res, "update was successful")
+    elif _type == "delete":
+        return Res.update(res, "delete was successful")
+    elif _type == "get":
+        return Res.fetch(res, "get was successful")
+    elif _type == "upload":
+        return Res.fetch(res, "upload complete")
+    elif _type == "token":
+        return Res.fetch(res, "token was generated")
+    else:
+        return Res.fetch(res, "response ok")
+
+
+def http_exception(res):
+    _return = None
+    # if isinstance(res, ResponseBadRequest):
+    #     _return = HTTPException(status_code=res.code, detail=res.message)
+    # elif isinstance(res, ResponseNotFound):
+    #     _return = HTTPException(status_code=res.code, detail=res.message)
+    # elif isinstance(res, ResponseUnauthorized):
+    #     _return = HTTPException(status_code=res.code, detail=res.message)
+    # elif isinstance(res, ResponseInternalServerError):
+    #     _return = HTTPException(status_code=res.code, detail=res.message)
+    if isinstance(res, SoftDeletedException):
+        _return = Res.delete_error(res, res.message)
+    elif isinstance(res, RecordNotFoundException):
+        _return = Res.not_found(res, res.message)
+    elif isinstance(res, ModifiedCountException):
+        _return = Res.write_error(res, res.message)
+    elif isinstance(res, ValidationError):
+        _return = Res.validation_error(res.errors())
+    elif isinstance(res, AssertionError):
+        _return = Res.validation_error(res, message=res)
+    elif isinstance(res, ValidationException):
+        _return = Res.validation_error(res.message, message=res.message)
+    elif isinstance(res, HTTPException):
+        _return = Res.bad_request(res.status_code, message=res.detail)
+    else:
+        _return = Res.server_error(res, f'unknown error: {str(res)} {type(res)}')
+    return _return

core/utils/router.py

@@ -0,0 +1,206 @@
+import time
+from functools import wraps
+from typing import Callable, TypeVar, Generic
+
+from fastapi import APIRouter, Request, Response, Body
+from fastapi import Security
+from fastapi.routing import APIRoute
+from starlette.background import BackgroundTasks
+
+from auth.utils import get_current_active_user
+from config import conf
+from core.utils import to_http_res, http_exception
+from core.utils.auth import get_current_active_user
+
+
+class BaseRoute(APIRoute):
+    def get_route_handler(self) -> Callable:
+        original_route_handler = super().get_route_handler()
+
+        async def custom_route_handler(request: Request) -> Response:
+            before = time.time()
+            response: Response = await original_route_handler(request)
+            duration = time.time() - before
+            response.headers["X-Response-Time"] = str(duration)
+            return response
+
+        return custom_route_handler
+
+
+def shrink(cal: Callable, name: str = None, bt: BackgroundTasks = None):
+    def can_inner(func):
+        @wraps(func)
+        async def wrapper(*args, **kwargs):
+            return await safe_route(name or cal.__name__, cal, *kwargs.values())
+        return wrapper
+
+    return can_inner
+
+
+def back_task(cal: Callable, name: str = None):
+    print(cal)
+
+    def back_inner(func):
+        print(func)
+
+        @wraps(func)
+        async def wrapper(*args, **kwargs):
+            if cal.__name__ == 'create_salam':
+                print(cal.__name__)
+            return await safe_route(name or cal.__name__, cal, *kwargs.values())
+
+        return wrapper
+
+    return back_inner
+
+
+def generate_router(
+        prefix,
+        dependencies=None,
+        auth_dependencies=False,
+        version='v1',
+        auth_scopes=['account']):
+    """
+    Generate a router with a prefix and dependencies.
+    Args:
+        prefix: prefix of the router.
+        dependencies: dependencies of the router.
+        auth_dependencies:  if router has auth dependencies is True.
+        version: version of the router.
+        auth_scopes: auth scopes of the router.
+
+    Returns: ApiRouter
+    """
+    if auth_dependencies:
+        dependencies = dependencies or []
+        if 'customer' in auth_scopes:
+            dependencies.extend(
+                [Security(get_current_active_customer, scopes=auth_scopes)])
+        else:
+            dependencies.extend(
+                [Security(get_current_active_user, scopes=auth_scopes)])
+    _pre = conf.server.prefix_url.replace('v1', version)
+    return APIRouter(
+        prefix=f"{_pre}/{prefix}",
+        route_class=BaseRoute,
+        dependencies=dependencies if dependencies else []
+    )
+
+
+async def safe_route(_type, fn, *args):
+    try:
+        return to_http_res(await fn(*args), _type)
+    except Exception as e:
+        return http_exception(e)
+
+
+class CrudMethods:
+    create = True
+    read = True
+    update = True
+    delete = True
+    get_all = True
+
+    def __init__(self, create=True, read=True, update=True, delete=True, fetch=True):
+        self.create = create
+        self.read = read
+        self.update = update
+        self.delete = delete
+        self.fetch = fetch
+
+    def __iter__(self):
+        r = []
+        if self.create:
+            r.append('create')
+        if self.read:
+            r.append('read')
+        if self.update:
+            r.append('update')
+        if self.delete:
+            r.append('delete')
+        if self.fetch:
+            r.append('fetch')
+        return r
+
+
+T = TypeVar("T")
+
+
+class SetupCrud(Generic[T]):
+    _url_prefix = {
+        'create': '/create',
+        'get': '/{_id}',
+        'update': '/{_id}',
+        'delete': '/{_id}',
+        'fetch': ''
+    }
+    _methods = {
+        'create': 'post',
+        'fetch': 'post',
+        'get': 'get',
+        'update': 'put',
+        'delete': 'delete',
+    }
+
+    def __init__(self,
+                 router: APIRouter,
+                 repository,
+                 model: T,
+                 tags=[],
+                 methods: CrudMethods = CrudMethods()
+                 ):
+
+        for i in ['create', 'update', 'get', 'delete', 'fetch']:
+            if i == 'create':
+                @router.post(
+                    self._url_prefix[i],
+                    response_model=model,
+                    tags=tags
+                )
+                async def create(body=Body(...)):
+                    b = T(**body)
+                    return await safe_route(i, getattr(repository, 'create'), b)
+            # elif i == 'update':
+            #     async def update(_id, body: T = Body(...)):
+            #         return await safe_route(i, getattr(repository, i), _id, body)
+            #
+            #     router.add_api_route(
+            #         _url_prefix[i],
+            #         update,
+            #         methods=[_methods[i]],
+            #         response_model=model,
+            #         tags=tags
+            #     )
+            # elif i == 'delete':
+            #     async def delete(_id):
+            #         return await safe_route(i, getattr(repository, i), _id)
+            #
+            #     router.add_api_route(
+            #         _url_prefix[i],
+            #         delete,
+            #         methods=[_methods[i]],
+            #         response_model=model,
+            #         tags=tags
+            #     )
+            # elif i == 'get':
+            #     async def get(_id):
+            #         return await safe_route(i, getattr(repository, i), _id)
+            #
+            #     router.add_api_route(
+            #         _url_prefix[i],
+            #         get,
+            #         methods=[_methods[i]],
+            #         response_model=model,
+            #         tags=tags
+            #     )
+            # elif i == 'fetch':
+            #     async def fetch(_id, body: T = Body(...)):
+            #         return await safe_route(i, getattr(repository, i), _id, body)
+            #
+            #     router.add_api_route(
+            #         _url_prefix[i],
+            #         fetch,
+            #         methods=[_methods[i]],
+            #         response_model=model,
+            #         tags=tags
+            #     )

core/utils/sumery/__init__.py

@@ -0,0 +1,89 @@
+"""
+smart upon mini query
+amazing mini query in any database
+"""
+
+from core.models.base import DELETED_AT_QUERY
+from core.utils import smart_cast
+from core.service.mongo_db import use_collection
+from core.exceptions import ValidationException
+
+
+class SumeryObject:
+    db: str
+    object_name: str
+    table_name: str
+    driver: str = 'mongo'
+    query_fn: callable
+    cols_search: list
+    key_col: str = '_id'
+    value_col: str
+    value_fn: callable
+    limit: 10
+
+    def __init__(self, db: str,
+                 object_name: str,
+                 cols_search: list,
+                 key_col: str,
+                 value_col: str,
+                 driver: str = 'mongo',
+                 table_name: str = None,
+                 query_fn: callable = None,
+                 limit: int = 10,
+                 value_fn: callable = None,
+                 ):
+        self.db = db
+        self.table_name = table_name
+        self.object_name = object_name
+        self.driver = driver
+        self.query_fn = query_fn
+        self.cols_search = cols_search
+        self.key_col = key_col
+        self.value_col = value_col
+        self.value_fn = value_fn
+        self.limit = limit
+
+
+__sumery_list: list[SumeryObject] = []
+
+
+async def sumery_executor(object_name: str, query: str, extra_filter: dict = None):
+    __query = query.strip()
+    if len(__query) < 3:
+        raise ValidationException('length of query not valid')
+    _so = next((x for x in __sumery_list if x.object_name == object_name), None)
+    if not _so:
+        raise ValidationException('summery object not found')
+    _query = {'$and': [{'$or': [{x: {'$regex': __query}} for x in _so.cols_search]},
+                       DELETED_AT_QUERY,
+                       extra_filter if extra_filter else {}]}
+    _projection = {
+        '_id': False,
+        'key': f'${_so.key_col}',
+        'value': f'${_so.value_col}'
+    }
+    records = use_collection(_so.table_name or _so.object_name, _so.db).find(_query, _projection).limit(_so.limit)
+    return [x async for x in records]
+
+
+async def sumery_default_getter(object_name: str, ids: list, extra_filter: dict = None):
+    if not len(ids):
+        raise ValidationException('length of ids not valid')
+    _so = next((x for x in __sumery_list if x.object_name == object_name), None)
+    if not _so:
+        raise ValidationException('summery object not found')
+    __cast_ids = [smart_cast(x) for x in ids]
+    _query = {'$and': [{_so.key_col: {'$in': __cast_ids}},
+                       DELETED_AT_QUERY,
+                       extra_filter if extra_filter else {}]}
+    _projection = {
+        '_id': False,
+        'key': f'${_so.key_col}',
+        'value': f'${_so.value_col}'
+    }
+    records = use_collection(_so.table_name or _so.object_name, _so.db).find(_query, _projection).limit(100)
+    return [x async for x in records]
+
+
+def setup_sumery_object(so: SumeryObject):
+    __sumery_list.append(so)

core/utils/table_engine/__init__.py

@@ -0,0 +1,4 @@
+from .engine import *
+from .filter import *
+from .input import *
+from .types import *

core/utils/table_engine/engine.py

@@ -0,0 +1,112 @@
+from pydantic import BaseModel
+from sqlalchemy import func
+
+from core.models.base import ORDER_BY_CREATED_AT
+from core.utils.table_engine.filter import TableEngineFilter
+from core.utils.table_engine.input import TableEngineInput
+
+
+class TableEngine:
+    object: any
+    input: TableEngineInput
+    is_sql_server: bool
+    is_deleted: bool
+    model: any
+    joins: list = None
+    filter: TableEngineFilter
+    sort: dict = ORDER_BY_CREATED_AT
+    extra_filter: list
+
+    def __init__(self, _object, _input: dict):
+        self.object = _object
+        _input = _input or {}
+        self.input = TableEngineInput(**_input)
+        self.input.get_limit()
+        self.filter = TableEngineFilter(self.input)
+        self.extra_filter = []
+
+    def soft_deleted(self):
+        self.is_deleted = True
+        self.input.is_deleted = True
+        return self
+
+    def sql_server(self):
+        self.is_sql_server = True
+        return self
+
+    def text_search_cols(self, cols: list):
+        self.input.text_search_cols(cols)
+        return self
+
+    def set_limit(self, limit: int):
+        self.input.p.s = limit
+        return self
+
+    def set_model(self, model: BaseModel.__class__):
+        self.model = model
+        return self
+
+    def join(self, joins):
+        self.joins = joins
+        return self
+
+    def sort_by(self, sort_by: dict | str):
+        self.sort = sort_by
+        return self
+
+    def _get_sql_order(self):
+        if isinstance(self.sort, dict):
+            return self.model.Id.desc()
+        return self.sort
+
+    def add_filter(self, filters):
+        self.extra_filter.append(filters)
+        self.input.extra_filter.append(filters)
+        return self
+
+    def get_query(self):
+        return self.input.dict()
+
+    async def exec(self):
+        if hasattr(self, 'is_sql_server') and self.is_sql_server:
+            return await self._exec_sql_server()
+        else:
+            return await self._exec_mongo()
+
+    async def _exec_mongo(self):
+        cursor = self.object.find(
+            self.filter.get()).sort(
+            **self.sort).limit(
+            self.input.get_limit()).skip(self.input.skip())
+        return [self.model(**document) async for document in cursor]
+
+    async def _exec_sql_server(self):
+        query = self.object \
+            .query(self.model)
+
+        if self.joins:
+            for j in self.joins:
+                if isinstance(j, list):
+                    query = query.join(*j)
+                else:
+                    query = query.join(j)
+
+        query = self.filter.get_sql_server(
+            self.model, query, self.extra_filter)
+
+        count = query.count()
+
+        query = query \
+            .order_by(self._get_sql_order()) \
+            .offset(self.input.skip()) \
+            .limit(self.input.get_limit())
+
+        return query.all(), count
+
+    async def count(self):
+        if hasattr(self, 'is_sql_server') and self.is_sql_server:
+            count = self.object.query(func.count(self.model.Id))
+        else:
+            count = await self.object.count_documents(
+                self.filter.get())
+        return count

core/utils/table_engine/filter.py

@@ -0,0 +1,160 @@
+__all__ = ['TableEngineFilter']
+
+from uuid import UUID
+
+from sqlalchemy import or_
+
+from core.utils.table_engine.input import TableEngineInput, FilterItem
+from core.utils.utils import is_valid_uuid
+
+
+def filter_item_generator(item: FilterItem) -> dict:
+    if is_valid_uuid(item.col):
+        return {
+            'data': {
+                "$elemMatch": {
+                    'field_id': UUID(item.col),
+                    'value': {**get_expression_filter(item.op, cast_value(item.val, item.type))}
+                }
+            }
+        }
+
+    return {item.col: {
+        **get_expression_filter(item.op, cast_value(item.val, item.type))}}
+
+
+def cast_value(val: str, type: str) -> any:
+    if type in ['NUMBER', 'STACK', 'FLOAT']:
+        return float(val)
+    if type == 'UUID':
+        return UUID(str(val))
+    return val
+
+
+def get_expression_filter(op: str, val: str) -> dict:
+    _op = get_operator(op)
+    match _op:
+        case "sw":
+            return {'$regex': f'^{val}'}
+        case "ew":
+            return {'$regex': f'{val}$'}
+        case "n_ew":
+            return {'$regex': f'(?<!{val})$'}
+        case "n_sw":
+            return {'$regex': f'^(?!{val})'}
+        case "n_like":
+            return {'$not': {'$regex': val}}
+        case "bet":
+            if type(val) is str:
+                return {'$gte': val.split('-')[0], '$lte': val.split('-')[1]}
+            return {'$gte': val[0], '$lte': val[1]}
+        case "n_bet":
+            if type(val) is str:
+                return {'$gte': val.split('-')[0], '$lte': val.split('-')[1]}
+            return {'$not': {'$gte': val[0], '$lte': val[1]}}
+        case "exi":
+            return {'$exists': True}
+        case "n_exi":
+            return {'$exists': False}
+        case "in":
+            return {'$exists': True}
+        case "not_in":
+            return {'$exists': False}
+    return {_op: val}
+
+
+def get_operator(op: str) -> str:
+    ops = {
+        "eq": "$eq",
+        "n_eq": "$ne",
+        "gt": "$gt",
+        "ge": "$ge",
+        "lt": "$lt",
+        "le": "$le",
+        "like": "$regex",
+        "n_like": "n_like",
+        "sw": "sw",
+        "ew": "ew",
+        'n_ew': 'n_ew',
+        'n_sw': 'n_sw',
+        'exi': 'exi',
+        "n_exi": "n_exi",
+        'lte': '$lte',
+        'gte': '$gte',
+        'in': 'in',
+        'n_in': 'not_in',
+        'bet': 'bet',
+        'n_bet': 'n_bet',
+    }
+
+    return ops[op] if op in ops else '$eq'
+
+
+def getAndOr(op: str) -> str:
+    return {"and": "$and", "or": '$or'}.get(op, '$and')
+
+
+class TableEngineFilter:
+    stack: list
+    input: TableEngineInput
+
+    def __init__(self, _input: TableEngineInput):
+        self.stack = []
+        self.input = _input
+
+    def add(self, filter_item: dict):
+        self.stack.append(filter_item)
+
+    def query_generator(self):
+        if not self.input.has_filter():
+            return []
+
+        filters = self.input.filters()
+        query = []
+        for filter_scope in filters:
+            query.append({getAndOr(filter_scope.op): [
+                filter_item_generator(item) for item in filter_scope.items]})
+        return query
+
+    def get(self):
+        f = self.stack
+        query = self.query_generator()
+        if query:
+            f.append(*query)
+        if self.input.has_text_search():
+            items = {}
+            for col in self.input.get_search_cols():
+                items[col] = {'$regex': self.input.get_text_search()}
+            f.append(items)
+        if hasattr(self.input, 'is_deleted') and self.input.is_deleted:
+            f.append({'deleted_at': {'$exists': False}})
+        if len(self.input.extra_filter) > 0:
+            f.extend(self.input.extra_filter)
+        if not f:
+            return {}
+        return {'$and': f}
+
+    def get_sql_server(self, model, query, filters):
+        """
+         text search filter
+        """
+        text_search_stack = []
+        fi = []
+        if self.input.has_text_search():
+            for col in self.input.get_search_cols():
+                if isinstance(col, str):
+                    text_search_stack.append(getattr(model, col).like(
+                        f'%{self.input.get_text_search()}%'))
+
+            if text_search_stack:
+                fi.append(or_(*text_search_stack))
+
+        if self.input.is_deleted:
+            fi.append(getattr(model, 'IsDeleted') == 0)
+
+        if filters:
+            for fil in filters:
+                fi.append(fil)
+
+        query = query.filter(*fi)
+        return query

core/utils/table_engine/input.py

@@ -0,0 +1,159 @@
+from enum import Enum
+from typing import List, Optional, Any
+
+import pydantic
+from pydantic import BaseModel
+
+
+class Cr(str, Enum):
+    OR = 'or'
+    AND = 'and'
+
+
+class FilterItem(BaseModel):
+    col: Optional[str]
+    cr: Optional[Cr]
+    id: Optional[str]
+    op: Optional[str]
+    type: Optional[str]
+    val: Optional[Any]
+
+    @staticmethod
+    def ValidItem(item: dict):
+        return item.get('col') and \
+            item.get('id') and \
+            item.get('op') and \
+            item.get('type')
+
+    @staticmethod
+    def ValidValue(item: dict):
+        return len(str(item.get('val', '')).strip()) > 0
+
+
+class TableEngineInput(BaseModel):
+
+    class FilterScope(BaseModel):
+        id: str
+        items: Optional[List[FilterItem]]
+        op: Optional[Cr]
+
+        def __init__(self, **kwargs):
+            super().__init__(**kwargs)
+            if kwargs.get('items'):
+                self.items = [
+                    FilterItem(
+                        **item) for item in filter(
+                        lambda i: FilterItem.ValidItem(i),
+                        kwargs.get(
+                            'items',
+                            []))]
+            self.op = Cr(kwargs.get('op'))
+            self.id = str(kwargs.get('id'))
+
+        @staticmethod
+        def ValidScope(item: dict):
+            return item.get('id') and item.get('items') and len(
+                item.get('items')) > 0 and item.get('op')
+
+        @staticmethod
+        def CleanItems(item):
+            clone = item.dict()
+            clone['items'] = list(filter(
+                lambda i: FilterItem.ValidValue(i), clone.get(
+                    'items', [])))
+            return TableEngineInput.FilterScope(**clone)
+
+    class Page(BaseModel):
+        c: int = 1
+        s: int = 5
+
+    class SearchModel(BaseModel):
+        text: Optional[str]
+        cols: Optional[List[Any]]
+
+    class SortModel(BaseModel):
+        field: Optional[str]
+        direction: Optional[str]
+
+    class Config:
+        extra = pydantic.Extra.ignore
+        anystr_strip_whitespace = True  # strip whitespaces from strings
+        underscore_attrs_are_private = False
+        arbitrary_types_allowed = True
+
+    f: Optional[List[FilterScope]]
+    extra_filter: list = []
+    o: Optional[SortModel]
+    p: Optional[Page] = Page()
+    s: Optional[SearchModel]
+    is_deleted: Optional[bool]
+
+    def limit(self):
+        return self.p.s
+
+    def order(self):
+        return self.p.o
+
+    def filters(self) -> List[FilterScope]:
+        return self.f
+
+    def has_filter(self) -> bool:
+        return self.f is not None and \
+            len(self.f) > 0 and \
+            self.f[0].items is not None and \
+            len(self.f[0].items) > 0
+
+    def has_extra_filter(self) -> bool:
+        return self.extra_filter is not None and \
+            len(self.extra_filter) > 0
+
+    def scopes(self):
+        return self.s
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        _input = {**{'p': {'c': 1, 's': 5},
+                     'f': [], 's': {}, 'o': {}}, **(kwargs or {})}
+        self.p = TableEngineInput.Page(**_input.get('p'))
+
+        if _input.get('f'):
+            self.f = [
+                TableEngineInput.FilterScope(
+                    **f) for f in filter(
+                    lambda fs:TableEngineInput.FilterScope.ValidScope(fs),
+                    _input.get('f'))]
+            self.f = list(map(
+                lambda fs: TableEngineInput.FilterScope.CleanItems(fs),
+                self.f))
+
+        if _input.get('s'):
+            self.s = TableEngineInput.SearchModel(**_input.get('s'))
+
+    def skip(self):
+        return self.page() * self.p.s
+
+    def page(self):
+        return self.p.c - 1
+
+    def virtual_page(self):
+        return self.page() + 1
+
+    def get_limit(self):
+        return self.p.s if self.p else 20
+
+    def text_search_cols(self, cols: List[str]):
+        if self.s:
+            self.s.cols = cols
+
+    def has_text_search(self) -> bool:
+        return self.s is not None and self.s.text is not None and self.s.cols is not None and len(
+            self.s.cols) > 0 and len(self.s.text.strip()) > 0
+
+    def add_filter(self, _filter):
+        self.extra_filter.append(_filter)
+
+    def get_search_cols(self) -> List[str]:
+        return self.s.cols if self.s else []
+
+    def get_text_search(self) -> str:
+        return self.s.text.strip() if self.s else None

core/utils/table_engine/types.py

@@ -0,0 +1,23 @@
+from typing import Generic, TypeVar, List, Optional
+
+from pydantic.generics import GenericModel
+
+T = TypeVar('T')
+
+
+class TableEngineInputType(GenericModel, Generic[T]):
+    f: Optional[List]
+    o: Optional[dict] = {}
+    p: Optional[dict] = {}
+    s: Optional[dict] = {}
+
+
+class TableEngineExport(GenericModel, Generic[T]):
+    rows: Optional[List[T]]
+    total: Optional[int]
+    page: Optional[int]
+    te: Optional[TableEngineInputType]
+
+
+class TableEngineRequestType(GenericModel, Generic[T]):
+    te: Optional[TableEngineInputType]

core/utils/te/__init__.py

@@ -0,0 +1,77 @@
+from typing import List
+
+from .engines import MsSqlEngine, MongoEngine, PgSqlEngine, AbstractEngine
+from .utils.Models import TableEngineInput
+from .utils.Models import WorkingMode, JoinModel
+from ..table_engine import TableEngineExport
+
+
+class NewTableEngine:
+    def __init__(self, query_class, model, inputs, mode, engine: AbstractEngine, extra_filter,
+                 default_sort_by, joins, search_cols, options=None, soft_deleted=False):
+        self.model = model
+        self.query_class = query_class
+        self.mode = mode
+        self.joins = False
+        self.raw_input = inputs
+        if isinstance(inputs, dict):
+            self.input = TableEngineInput(**inputs if inputs else {})
+        else:
+            self.input = TableEngineInput(**inputs.dict() if inputs else {})
+        self.input.extra_filter = extra_filter
+        self.default_sort_by = default_sort_by
+        self.joins = joins
+        self.soft_deleted = soft_deleted
+        self.options = options
+        self.search_cols = search_cols
+        self.engine = engine(self)
+
+    @staticmethod
+    def mongo(query_class,
+              model,
+              inputs,
+              mode: WorkingMode = WorkingMode.Development,
+              extra_filter=None,
+              soft_deleted=False,
+              default_sort_by=None, joins=None, search_cols=[]):
+        return NewTableEngine(query_class, model, inputs, mode, engine=MongoEngine, extra_filter=extra_filter,
+                              default_sort_by=default_sort_by, joins=joins, search_cols=search_cols,
+                              soft_deleted=soft_deleted)
+
+    @staticmethod
+    def mssql(query_class, model, inputs, mode: WorkingMode = WorkingMode.Development, extra_filter=None,
+              default_sort_by=None, joins=None, search_cols=[], options=None):
+        return NewTableEngine(query_class, model, inputs, mode, engine=MsSqlEngine, extra_filter=extra_filter,
+                              default_sort_by=default_sort_by, joins=joins, search_cols=search_cols, options=options)
+
+    @staticmethod
+    def pgsql(query_class, model, inputs, mode: WorkingMode = WorkingMode.Development, extra_filter=None,
+              default_sort_by=None, joins=None, search_cols=None, options=None):
+        return NewTableEngine(query_class, model, inputs, mode, engine=PgSqlEngine, extra_filter=extra_filter,
+                              default_sort_by=default_sort_by, joins=joins, search_cols=search_cols, options=options)
+
+    async def exe(self):
+        return await self.engine.exe()
+
+    async def to_dict(self) -> TableEngineExport:
+        rows, count = await self.exe()
+        self.input.extra_filter = None
+        del self.engine
+        return TableEngineExport(
+            rows=rows,
+            page=self.input.virtual_page(),
+            total=count,
+            te=self.input
+        )
+
+    def add_extra_filter(self, filters):
+        self.input = filters
+        return self
+
+    def joins(self, joins: List[JoinModel]):
+        self.joins = joins
+        return self
+
+    def set_custom_entities(self, entities=None):
+        self.engine.custom_entities = entities
+        return self

core/utils/te/engines/AbstractEngine.py

@@ -0,0 +1,51 @@
+import abc
+
+__all__ = "AbstractEngine"
+
+
+class AbstractEngineClass(abc.ABC):
+    def __init__(self, table_engine):
+        self.model = table_engine.model
+        self.query_class = table_engine.query_class
+        self.input = table_engine.input
+        self.filter = table_engine.input.f
+        self.default_sort_by = table_engine.default_sort_by
+        self.custom_entities = []
+        self.joins = table_engine.joins
+        self.options = table_engine.options
+        self.search_cols = table_engine.search_cols
+        self.soft_deleted = table_engine.soft_deleted
+
+    @abc.abstractmethod
+    def exe(self):
+        pass
+
+    @abc.abstractmethod
+    def set_filters(self):
+        pass
+
+    @abc.abstractmethod
+    def set_sort(self):
+        pass
+
+    @staticmethod
+    @abc.abstractmethod
+    def get_expression_filter(col=None, op: str = 'eq', val: str = '') -> dict:
+        pass
+
+    @staticmethod
+    @abc.abstractmethod
+    def get_operator(op: str):
+        pass
+
+    def count(self):
+        pass
+
+    @staticmethod
+    @abc.abstractmethod
+    def generate_scope_filter(filters: list, general=False):
+        pass
+
+    @abc.abstractmethod
+    def set_joins(self):
+        pass

core/utils/te/engines/Mongo.py

@@ -0,0 +1,187 @@
+import datetime
+from typing import Union, List
+from uuid import UUID
+
+from .AbstractEngine import AbstractEngineClass
+from ..utils.Models import FilterItem, FilterScope
+
+
+class MongoEngine(AbstractEngineClass):
+    rows = []
+    stack = []
+    filters = {}
+
+    async def exe(self):
+        _f = self.filters = self.set_filters()
+        tmp = self.query_class.find(_f).sort(**self.set_sort())
+        self.rows = tmp.limit(
+            self.input.get_limit()).skip(self.input.skip())
+        return [self.model(**document) async for document in self.rows], await self.count()
+
+    def query_generator(self):
+        if not self.input.has_filter():
+            return []
+        filters = self.input.filters()
+        query = self.generate_scope_filter(filters, general=True)
+        return query
+
+    def set_filters(self):
+        filters = []
+        query = self.query_generator()
+        if query:
+            filters.append(query)
+        if self.input.has_text_search():
+            items = []
+            _search_cols = self.input.get_search_cols() if (
+                    self.input.get_search_cols() and len(self.input.get_search_cols())) else self.search_cols
+            if _search_cols:
+                for col in _search_cols:
+                    items.append({col: {'$regex': self.input.get_text_search()}})
+                filters.append({"$or": items})
+        if self.input.extra_filter:
+            filters.extend(self.input.extra_filter)
+        if self.soft_deleted:
+            filters.append({'deleted_at': {'$exists': False}})
+        if not filters:
+            return {}
+        self.filters = {'$and': filters}
+        return self.filters
+
+    def set_sort(self):
+        if self.input.o.field:
+            return {'key_or_list': self.input.o.field,
+                    'direction': 1 if self.input.o.direction == "asc" else -1}
+        elif self.default_sort_by:
+            return self.default_sort_by
+        else:
+            return {'key_or_list': 'created_at', 'direction': -1}
+
+    @staticmethod
+    def filter_item_generator(item: FilterItem) -> dict:
+        if MongoEngine.is_valid_uuid(str(item.col)):
+            return {
+                'data': {
+                    "$elemMatch": {
+                        'field_id': UUID(item.col),
+                        'value': {
+                            **MongoEngine.get_expression_filter(item.op, MongoEngine.cast_value(item.val, item.type), )}
+                    }
+                }
+            }
+        return {item.col: {
+            **MongoEngine.get_expression_filter(col=item.col, op=item.op,
+                                                val=MongoEngine.cast_value(item.val, item.type), )}}
+
+    @staticmethod
+    def is_valid_uuid(uuid_to_test, version=4):
+        try:
+            uuid_obj = UUID(str(uuid_to_test), version=version)
+        except ValueError:
+            return False
+        return str(uuid_obj) == str(uuid_to_test)
+
+
+    @staticmethod
+    def cast_value(val: Union[str, List], type: str) -> any:
+        if type in ['NUMBER', 'STACK', 'FLOAT']:
+            if isinstance(val, List):
+                return [float(x) for x in val]
+            return float(val)
+        if type in ['DATE', 'DATETIME']:
+            if isinstance(val, List):
+                return [datetime.datetime.fromisoformat(str(x)) for x in val]
+            return datetime.datetime.fromisoformat(val)
+        if type == 'UUID':
+            if isinstance(val, List):
+                return [UUID(str(x)) for x in val]
+            return UUID(str(val))
+        return val
+
+
+    @staticmethod
+    def get_expression_filter(col=None, op: str = 'eq', val: str = '') -> dict:
+        _op = MongoEngine.get_operator(op)
+        if _op == "sw":
+            return {'$regex': f'^{val}'}
+        elif _op == "ew":
+            return {'$regex': f'{val}$'}
+        elif _op == "n_ew":
+            return {'$regex': f'(?<!{val})$'}
+        elif _op == "n_sw":
+            return {'$regex': f'^(?!{val})'}
+        elif _op == "n_like":
+            return {'$not': {'$regex': val}}
+        elif _op == "bet":
+            if isinstance(val, str):
+                return {'$gte': val.split('-')[0], '$lte': val.split('-')[1]}
+            return {'$gte': val[0], '$lte': val[1]}
+        elif _op == "n_bet":
+            if isinstance(val, str):
+                return {'$gte': val.split('-')[0], '$lte': val.split('-')[1]}
+            return {'$not': {'$gte': val[0], '$lte': val[1]}}
+        elif _op == "exi":
+            return {'$exists': True}
+        elif _op == "n_exi":
+            return {'$exists': False}
+        elif _op == "in":
+            return {'$exists': True}
+        elif _op == "not_in":
+            return {'$exists': False}
+        return {_op: val}
+
+    @staticmethod
+    def get_operator(op: str) -> str:
+        ops = {
+            "eq": "$eq",
+            "n_eq": "$ne",
+            "gt": "$gt",
+            "ge": "$ge",
+            "lt": "$lt",
+            "le": "$le",
+            "like": "$regex",
+            "n_like": "n_like",
+            "sw": "sw",
+            "ew": "ew",
+            'n_ew': 'n_ew',
+            'n_sw': 'n_sw',
+            'exi': 'exi',
+            "n_exi": "n_exi",
+            'lte': '$lte',
+            'gte': '$gte',
+            'in': 'in',
+            'n_in': 'not_in',
+            'bet': 'bet',
+            'n_bet': 'n_bet',
+        }
+        return ops[op] if op in ops else '$eq'
+
+    async def count(self):
+        return await self.query_class.count_documents(self.filters)
+
+    @staticmethod
+    def generate_scope_filter(filters: list, general=False):
+        if len(filters) == 1:
+            _o = filters[0]
+            if isinstance(_o, FilterItem):
+                return MongoEngine.filter_item_generator(filters[0])
+            elif isinstance(_o, FilterScope):
+                return MongoEngine.generate_scope_filter(_o.items)
+            else:
+                return _o
+        else:
+            _a = filters.pop(0)
+            _a = (MongoEngine.filter_item_generator(_a) if isinstance(_a,
+                                                                      FilterItem) else _a) if not general else (
+                MongoEngine.generate_scope_filter(_a.items) if isinstance(_a, FilterScope) else _a)
+            _b = filters.pop(0)
+            op = '$' + (_b.cr if not general else _b.op)
+            _b = (MongoEngine.filter_item_generator(_b) if isinstance(_b,
+                                                                      FilterItem) else _b) if not general else (
+                MongoEngine.generate_scope_filter(_b.items) if isinstance(_b, FilterScope) else _b)
+            filters = [{op: [
+                _a, _b
+            ]}] + filters
+            return MongoEngine.generate_scope_filter(filters, general)
+
+    def set_joins(self):
+        pass

core/utils/te/engines/Mssql.py

@@ -0,0 +1,146 @@
+from sqlalchemy import or_, and_, desc, asc
+from sqlalchemy.sql.elements import BooleanClauseList, BinaryExpression, UnaryExpression
+
+from .AbstractEngine import AbstractEngineClass
+from ..utils.Models import FilterItem, FilterScope
+
+
+class MsSqlEngine(AbstractEngineClass):
+
+    def set_joins(self):
+        pass
+
+    filters = {}
+    rows = []
+    sort = None
+
+    async def exe(self):
+        _f = self.set_filters()
+        if not self.custom_entities:
+            query = self.query_class.query(self.model)
+        else:
+            query = self.query_class.query(*self.custom_entities)
+        if self.joins:
+            for j in self.joins:
+                if isinstance(j, list):
+                    query = query.join(*j)
+                else:
+                    query = query.join(j)
+
+        if self.options:
+            query = query.options(*self.options)
+        tmp = query.filter(_f).order_by(self.set_sort())
+        self.rows = tmp.limit(self.input.get_limit()).offset(self.input.skip()).all()
+        return self.rows, tmp.count(), tmp
+
+    def query_generator(self):
+        if not self.input.has_filter():
+            return None
+        filters = self.input.filters()
+        query = self.generate_scope_filter(filters, general=True)
+        return query
+
+    def set_sort(self):
+        if self.input.o.field:
+            self.sort = asc(getattr(self.model, self.input.o.field)) if self.input.o.direction == 'asc' else desc(
+                getattr(self.model, self.input.o.field))
+            return self.sort
+        elif isinstance(self.default_sort_by, UnaryExpression):
+            return self.default_sort_by
+        else:
+            return True
+
+    def set_filters(self):
+        filters = []
+        query = self.query_generator()
+        if isinstance(query, BooleanClauseList) or isinstance(query, BinaryExpression):
+            filters.append(query)
+        if self.input.has_text_search():
+            items = []
+            _search_cols = self.input.get_search_cols() if (
+                    self.input.get_search_cols() and len(self.input.get_search_cols())) else self.search_cols
+            if _search_cols:
+                for col in _search_cols:
+                    has_join = len(col.split('.')) > 1
+                    if has_join:
+                        pass
+                    else:
+                        items.append(
+                            getattr(self.model, col).ilike("%{search}%".format(search=self.input.get_text_search())))
+                filters.append(or_(*items))
+        if self.input.extra_filter:
+            for f in self.input.extra_filter:
+                filters.append(f)
+        if not filters:
+            return True
+        self.filters = and_(*filters)
+        return self.filters
+
+    @staticmethod
+    def get_expression_filter(col=None, op: str = '', val: str = '') -> dict:
+        _op = MsSqlEngine.get_operator(op)
+        return getattr(col, op)(val)
+
+    @staticmethod
+    def get_operator(op: str):
+        ops = {
+            "eq": "__eq__",
+            "n_eq": "__ne__",
+            "gt": "__gt__",
+            "gte": "__ge__",
+            "lte": "__lte__",
+            "lt": "__lt__",
+            "like": "ilike",
+            "n_like": "notlike",
+            "in": "in_",
+            "n_in": "not_in",
+            "bet": "between",
+            "n_bet": "not_between",
+            "sw": "startswith",
+            "ew": "endswith",
+            'n_ew': 'not_endswith',
+            'n_sw': 'not_startswith',
+            'exi': 'exi',
+            "n_exi": "n_exi",
+        }
+        return ops[op] if op in ops else '__eq__'
+
+    @staticmethod
+    def get_and_or(op):
+        return or_ if op == 'or' else and_
+
+    def count(self):
+        pass
+
+    def filter_item_generator(self, item: FilterItem):
+        has_join = len(item.col.split('.')) > 1
+        if has_join:
+
+            pass
+        else:
+            return getattr(getattr(self.model, item.col), self.get_operator(item.op))(item.val)
+
+    def generate_scope_filter(self, filters: list, general=False):
+        if len(filters) == 1:
+            _o = filters[0]
+            if isinstance(_o, FilterItem):
+                return self.filter_item_generator(filters[0])
+            elif isinstance(_o, FilterScope):
+                return self.generate_scope_filter(_o.items)
+            else:
+                return _o
+        else:
+            _a = filters.pop(0)
+            _a = (self.filter_item_generator(_a) if isinstance(_a,
+                                                               FilterItem) else _a) if not general else (
+                self.generate_scope_filter(_a.items) if isinstance(_a, FilterScope) else _a)
+
+            _b = filters.pop(0)
+
+            _op = MsSqlEngine.get_and_or(_b.cr if not general else _b.op)
+            _b = (self.filter_item_generator(_b) if isinstance(_b,
+                                                               FilterItem) else _b) if not general else (
+                self.generate_scope_filter(_b.items) if isinstance(_b, FilterScope) else _b)
+
+            filters = [_op(_a, _b)] + filters
+            return self.generate_scope_filter(filters, general)

core/utils/te/engines/Pgsql.py

@@ -0,0 +1,13 @@
+from .AbstractEngine import AbstractEngineClass
+
+
+class PgSqlEngine(AbstractEngineClass):
+
+    def exe(self):
+        pass
+
+    def get_filters(self):
+        pass
+
+    def pagination(self):
+        pass

core/utils/te/engines/__init__.py

@@ -0,0 +1,4 @@
+from enum import Enum
+from .Mssql import MsSqlEngine
+from .Pgsql import PgSqlEngine
+from .Mongo import MongoEngine

core/utils/te/utils/Encoder.py

@@ -0,0 +1,19 @@
+import json
+
+from sqlalchemy.orm import DeclarativeMeta
+
+
+class AlchemyEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj.__class__, DeclarativeMeta):
+            fields = {}
+            for field in [x for x in dir(obj) if not x.startswith('_') and x != 'metadata']:
+                data = obj.__getattribute__(field)
+                try:
+                    json.dumps(data)
+                    fields[field] = data
+                except TypeError:
+                    fields[field] = None
+            return fields
+
+        return json.JSONEncoder.default(self, obj)

core/utils/te/utils/Models.py

@@ -0,0 +1,160 @@
+from enum import Enum
+from typing import Optional, Any, List
+
+import pydantic
+from pydantic import BaseModel
+
+
+class FilterItem(BaseModel):
+    col: Optional[str]
+    cr: Optional[str]
+    id: Optional[str]
+    op: Optional[str]
+    type: Optional[str]
+    val: Optional[Any]
+
+    @staticmethod
+    def ValidItem(item: dict):
+        return item.get('col') and \
+            item.get('id') and \
+            item.get('op') and \
+            item.get('type')
+
+    @staticmethod
+    def ValidValue(item: dict):
+        return len(str(item.get('val', '')).strip()) > 0
+
+
+class Cr(str, Enum):
+    OR = 'or'
+    AND = 'and'
+
+
+class FilterScope(BaseModel):
+    id: str
+    items: Optional[List[FilterItem]]
+    op: Optional[Cr]
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        if kwargs.get('items'):
+            self.items = [
+                FilterItem(**item) for item in filter(lambda i: FilterItem.ValidItem(i), kwargs.get('items', []))]
+        self.op = Cr(kwargs.get('op'))
+        self.id = str(kwargs.get('id'))
+
+    @staticmethod
+    def ValidScope(item: dict):
+        return item.get('id') and item.get('items') and len(
+            item.get('items')) > 0 and item.get('op')
+
+    @staticmethod
+    def CleanItems(item):
+        clone = item.dict()
+        clone['items'] = list(filter(
+            lambda i: FilterItem.ValidValue(i), clone.get(
+                'items', [])))
+        return FilterScope(**clone)
+
+
+class Page(BaseModel):
+    c: int = 1
+    s: int = 5
+
+
+class SearchModel(BaseModel):
+    text: Optional[str]
+    cols: Optional[List[Any]]
+
+
+class SortModel(BaseModel):
+    field: Optional[str]
+    direction: Optional[str]
+
+
+class JoinModel(BaseModel):
+    model: BaseModel
+    show: List[str]
+    search: Optional[List[str]]
+
+
+class Config:
+    extra = pydantic.Extra.ignore
+    any_str_strip_whitespace = True  # strip whitespaces from strings
+    underscore_attrs_are_private = False
+    arbitrary_types_allowed = True
+
+
+class TableEngineInput(BaseModel):
+    f: Optional[List[FilterScope]]
+    extra_filter: list = []
+    o: Optional[SortModel]
+    p: Optional[Page] = Page()
+    s: Optional[SearchModel]
+    is_deleted: Optional[bool]
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        _input = {**{'p': {'c': 1, 's': 5},
+                     'f': [], 's': {}, 'o': {}}, **(kwargs or {})}
+        self.p = Page(**_input.get('p'))
+        self.o = SortModel(**_input.get('o'))
+        if _input.get('f'):
+            self.f = [FilterScope(**f) for f in
+                      filter(lambda fs: FilterScope.ValidScope(fs), _input.get('f'))]
+            self.f = list(map(lambda fs: FilterScope.CleanItems(fs), self.f))
+        if _input.get('s'):
+            self.s = SearchModel(**_input.get('s'))
+
+    def limit(self):
+        return self.p.s
+
+    def order(self):
+        return self.o
+
+    def filters(self) -> List[FilterScope]:
+        return self.f
+
+    def has_filter(self) -> bool:
+        return self.f is not None and \
+            len(self.f) > 0 and \
+            self.f[0].items is not None and \
+            len(self.f[0].items) > 0
+
+    def has_extra_filter(self) -> bool:
+        return self.extra_filter is not None and \
+            len(self.extra_filter) > 0
+
+    def skip(self):
+        return self.page() * self.p.s
+
+    def page(self):
+        return self.p.c - 1
+
+    def virtual_page(self):
+        return self.page() + 1
+
+    def get_limit(self):
+        limit = self.p.s if self.p else 20
+        return limit if limit < 100 else 100
+
+    def text_search_cols(self, cols: List[str]):
+        if self.s:
+            self.s.cols = cols
+
+    def has_text_search(self) -> bool:
+        return self.s is not None and self.s.text is not None and len(self.s.text.strip()) > 0
+
+    def add_filter(self, _filter):
+        self.extra_filter.append(_filter)
+
+    def get_search_cols(self) -> List[str]:
+        return self.s.cols if self.s else []
+
+    def get_text_search(self) -> str:
+        return self.s.text.strip() if self.s else None
+
+
+class WorkingMode(Enum):
+    Development = "development"
+    Production = "production"

core/utils/utils.py

@@ -0,0 +1,150 @@
+import re
+from datetime import datetime
+from functools import reduce
+from time import time
+from uuid import uuid4, UUID
+
+import pytz as pytz
+import shortuuid
+from fastapi.requests import Request
+
+
+def get_time() -> datetime:
+    """
+    Returns the current time.
+    Returns:
+    """
+    return datetime.fromtimestamp(time(), tz=pytz.timezone('Asia/Tehran'))
+
+
+def get_uuid() -> UUID:
+    """Returns an unique UUID (UUID4)"""
+    return uuid4()
+
+
+def is_numeric(_input) -> bool:
+    """Returns True if the input is a number."""
+    try:
+        val = int(_input)
+        return True
+    except ValueError:
+        return False
+
+
+def get_v_id(prefix: str, prefix_len=2, source_id: UUID = None) -> str:
+    main_id = source_id if source_id is not None else uuid4()
+    short_id = shortuuid.encode(main_id)
+    return str(prefix)[0:prefix_len] + '.' + short_id[:7]
+
+
+def get_qr_v_id(source_id: UUID = None) -> str:
+    main_id = source_id if source_id is not None else uuid4()
+    short_id = shortuuid.encode(main_id)
+    return short_id[:8]
+
+
+def is_valid_uuid(uuid_to_test, version=4):
+    """
+    Check if uuid_to_test is a valid UUID.
+
+     Parameters
+    ----------
+    uuid_to_test : str
+    version : {1, 2, 3, 4}
+
+     Returns
+    -------
+    `True` if uuid_to_test is a valid UUID, otherwise `False`.
+
+     Examples
+    --------
+    >>> is_valid_uuid('c9bf9e57-1685-4c89-bafb-ff5af830be8a')
+    True
+    >>> is_valid_uuid('c9bf9e58')
+    False
+    """
+
+    try:
+        uuid_obj = UUID(str(uuid_to_test), version=version)
+    except ValueError:
+        return False
+    return str(uuid_obj) == str(uuid_to_test)
+
+
+def get_base_url(request: Request) -> str:
+    """
+    Returns the base url of the request.
+
+    Parameters
+    ----------
+    request : Request
+
+    Returns
+    -------
+    str
+    """
+    return '{0}://{1}{2}/'.format(request.url.scheme,
+                                  request.url.hostname,
+                                  ':' + request.url.port.__str__() if request.url.port else '')
+
+
+def p2e_numbers(input: str) -> str:
+    """
+    convert persian numbers to english numbers.
+    """
+    _map = {
+        '۱': '1',
+        '۲': '2',
+        '۳': '3',
+        '۴': '4',
+        '۵': '5',
+        '۶': '6',
+        '۷': '7',
+        '۸': '8',
+        '۹': '9',
+        '۰': '0'
+    }
+    a = map(lambda ch: _map[ch] if ch in _map else ch, input)
+    return ''.join(list(a))
+
+
+def is_list_of_strings(lst):
+    return bool(lst) and isinstance(lst, list) and all(isinstance(elem, str) for elem in lst)
+
+
+def smart_cast(_input):
+    if is_numeric(_input):
+        return int(_input)
+    if is_valid_uuid(_input):
+        return UUID(str(_input))
+    return _input
+
+
+def is_mobile_number(mobile: str):
+    regex = r"^\+?[0-0][0-9]{10}$"
+    if mobile and not re.search(regex, mobile, re.I):
+        return False
+    return True
+
+
+def is_email(email: str):
+    regex = r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b'
+    if email and not re.search(regex, email, re.I):
+        return False
+    return True
+
+
+def deep_get(dictionary, keys, default=None):
+    return reduce(lambda d, key: d.get(key, default) if isinstance(d, dict) else default, keys.split("."), dictionary)
+
+
+def get_tag_strips(tag_name: str, content: str):
+    try:
+        r_string = fr'\[{tag_name}\](.|\n)*?\[\/{tag_name}\]'
+        matches = re.search(r_string, str(content), re.MULTILINE)
+        raw = matches.group(0)
+        raw = raw.replace(f'[{tag_name}]', '')
+        raw = raw.replace(f'[/{tag_name}]', '')
+        return raw
+    except BaseException:
+        return False