Initial commit

core/utils/__init__.py

@@ -0,0 +1,8 @@
+import os
+
+from .utils import *
+from .response import *
+
+
+def get_path(__file__=__file__):
+    return os.path.abspath(os.path.dirname(__file__)) + '/'

core/utils/api.py

@@ -0,0 +1,5 @@
+from config import conf
+
+
+def get_namespace_api(prefix):
+    return f"{conf.server.prefix_url}/{prefix}"

core/utils/app_context.py

@@ -0,0 +1,21 @@
+from starlette.requests import Request
+
+from core.app import app
+from core.utils.auth import UserModel
+
+
+def use_request() -> Request:
+    try:
+        return app.state.request.state
+    except AttributeError:
+        raise AttributeError("No request object found in app.state")
+
+
+def use_user(user=None) -> UserModel:
+    try:
+        if user is None:
+            return use_request().user
+        else:
+            app.state.request.state.user = user
+    except AttributeError:
+        raise AttributeError("No user object found in app.state")

core/utils/auth/__init__.py

@@ -0,0 +1,78 @@
+import httpx
+from fastapi import HTTPException, Depends, Security
+from fastapi.security import SecurityScopes
+from httpx import ConnectError
+from jose import jwt
+from starlette.requests import Request
+
+from config import conf
+from .model import UserModel, ValidateTokenInput
+from .schema import oauth2
+from ..app_context import use_user
+from ...service.redis import redis_set, redis_get
+
+
+def decode_token(token: str):
+    return jwt.decode(
+        token.split(' ').pop(), conf.auth.secret_key, algorithms=[conf.auth.algorithm])
+
+
+async def validate_token(_input: ValidateTokenInput):
+    try:
+        _cache_key = _input.token.split('.')[-1]
+        _cache_auth = await redis_get(f'{conf.auth.cache_prefix_key}{_cache_key}', conf.database.get('auth_redis'))
+        if not _cache_auth:
+            client = httpx.AsyncClient()
+            headers = {'Content-Type': 'application/json', }
+
+            result = await client.request('POST', conf.auth.url, json=_input.dict(), headers=headers)
+            export = result.text
+            if 'application/json' in result.headers['content-type']:
+                export = result.json()
+
+            if result.status_code > 299:
+                raise HTTPException(status_code=result.status_code, detail=export)
+
+            if 'v2' in conf.auth.url:
+                export = export.get('data')
+
+            await redis_set(f'{conf.auth.cache_prefix_key}{_cache_key}', export, conf.database.get('auth_redis'),
+                            ex=conf.auth.cache_ttl)
+
+            return UserModel(**export)
+        else:
+            return UserModel(**_cache_auth)
+
+    except ConnectError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    except HTTPException as e:
+        raise e
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+
+async def get_current_user(security_scopes: SecurityScopes, token: str = Depends(oauth2)):
+    if security_scopes.scopes:
+        authenticate_value = f'Bearer scope="{security_scopes.scope_str}"'
+    else:
+        authenticate_value = f"Bearer"
+    credentials_exception = HTTPException(
+        status_code=401,
+        detail="Could not validate credentials",
+        headers={"WWW-Authenticate": authenticate_value},
+    )
+    user = await validate_token(ValidateTokenInput(token=token, scopes=security_scopes.scopes))
+    if user is None:
+        raise credentials_exception
+    user.scopes = security_scopes.scopes if isinstance(
+        security_scopes.scopes, list) else security_scopes.scopes
+    return user
+
+
+async def get_current_active_user(request: Request,
+                                  current_user: UserModel = Security(get_current_user, scopes=["account"]), ):
+    if current_user.disabled:
+        raise HTTPException(status_code=400, detail="Inactive user")
+    request.state.user = UserModel(**current_user.mongo_dict())
+    use_user(UserModel(**current_user.mongo_dict()))
+    return current_user

core/utils/auth/model.py

@@ -0,0 +1,29 @@
+import uuid
+from typing import Optional, Union, List
+
+import pydantic
+from pydantic import Field
+
+from core.models.base import BaseModel
+
+
+class UserModel(BaseModel):
+    id: Optional[Union[uuid.UUID, str]] = Field(alias='_id')
+    username: Optional[str]
+    mobile: Optional[str]
+    name: Optional[str]
+    first_name: Optional[str]
+    last_name: Optional[str]
+    is_su: Optional[bool]
+    unit: Optional[str]
+    avatar: Optional[str]
+    desc: Optional[str]
+    scopes: Optional[List[str]]
+    home_path: Optional[str]
+    roles: Optional[List[uuid.UUID]]
+    disabled: Optional[bool]
+
+
+class ValidateTokenInput(pydantic.BaseModel):
+    token: str
+    scopes: Optional[List[str]] = ['account']

core/utils/auth/schema.py

@@ -0,0 +1,9 @@
+from fastapi.security import OAuth2PasswordBearer
+
+oauth2 = OAuth2PasswordBearer(
+    tokenUrl="token",
+    scopes={
+        "account": "Read information about the current user.",
+        "scm": "Scm Modules",
+    },
+)

core/utils/can.py

@@ -0,0 +1,18 @@
+from functools import wraps
+from typing import List, Union
+
+from core.utils.app_context import use_user
+
+
+def can(permissions: Union[str, List[str]]):
+    def can_inner(func):
+        @wraps(func)
+        async def wrapper(*args, **kwargs):
+            print(permissions)
+            user = use_user()
+            # raise HTTPException(status_code=403, detail='forbidden request')
+            return await func(*args, **kwargs)
+
+        return wrapper
+
+    return can_inner

core/utils/crudify.py

@@ -0,0 +1,118 @@
+from dataclasses import dataclass
+from uuid import UUID
+
+from core.exceptions import ModifiedCountException, RecordNotFoundException
+from core.models.base import BaseModel, DELETED_AT_QUERY
+from core.service.mongo_db import use_collection
+from core.utils import get_time, get_v_id
+
+
+@dataclass
+class ConfigMongoWriter:
+    created_at: bool = True
+    updated_at: bool = True
+    has_owner: bool = True
+    init_id: bool = True
+    init_v_id: bool = False
+
+
+async def mongo_inserter(model: BaseModel, collection: str, options: ConfigMongoWriter = ConfigMongoWriter(),
+                         getter=None, db=None, v_id_prefix=None, client=None) -> dict | BaseModel:
+    """
+    Inserts data into collection
+    """
+    try:
+        if options.init_id:
+            model.init_id()
+        if options.has_owner:
+            model.init_owner()
+        if options.created_at and options.updated_at:
+            model.cu_at()
+        if options.created_at:
+            model.c_at()
+        if options.updated_at:
+            model.u_at()
+        if options.init_v_id:
+            model.v_id = get_v_id(v_id_prefix or 'uk', source_id=model.id)
+
+        document = model.mongo_dict()
+        if client:
+            result = await client(collection).insert_one(document)
+        else:
+            result = await use_collection(collection, db).insert_one(document)
+        assert result.acknowledged
+        if getter:
+            return await getter(result.inserted_id)
+        return result
+    except AssertionError as e:
+        raise AssertionError(f'{collection} insert error')
+    except Exception as e:
+        raise e
+
+
+async def mongo_soft_deleter(query: dict | UUID, collection: str, db=None, client=None):
+    """
+    Updates data in collection
+    """
+    # doc = {"$set": {'deleted_at': get_time()}, }
+    _query = query
+    if isinstance(query, UUID):
+        _query = {'_id': query}
+    __trash_collection = f'{collection}--trash'
+    __driver = client or use_collection
+    __current_record = await __driver(collection, db=db).find_one(_query)
+    if not __current_record:
+        raise RecordNotFoundException(identifier=str(query))
+    __current_record['deleted_at'] = get_time()
+    __driver(__trash_collection, db=db).insert_one(__current_record)
+    __driver(collection, db=db).delete_one(_query)
+    return True
+
+
+async def mongo_updater(model: BaseModel,
+                        query: dict | UUID,
+                        collection: str,
+                        options: ConfigMongoWriter = ConfigMongoWriter(created_at=False, has_owner=False),
+                        getter=None,
+                        db=None,
+                        extra_query: dict = None,
+                        client=None,
+                        ):
+    """
+    Updates data in collection
+    """
+    _query = query
+    if isinstance(query, UUID):
+        _query = {'_id': query}
+    if options.has_owner:
+        model.init_owner()
+    if options.updated_at:
+        model.u_at()
+    if client:
+        result = await client(collection).update_one(_query, {"$set": model.dict(),
+                                                              **(extra_query if extra_query else {})})
+    else:
+        result = await use_collection(collection, db).update_one(_query, {"$set": model.dict(),
+                                                                          **(extra_query if extra_query else {})})
+    if not result.modified_count:
+        raise ModifiedCountException(query)
+    if getter:
+        return await getter(_query)
+    return result
+
+
+async def mongo_find_one(query: dict | UUID, collection: str, db=None, client=None, project=None) -> dict:
+    """
+     Ge
+     """
+    _query = query
+    if isinstance(query, UUID):
+        _query = {'_id': query}
+    _query = dict(DELETED_AT_QUERY, **_query)
+    if client:
+        result = await client(collection).find_one(_query, project)
+    else:
+        result = await use_collection(collection, db).find_one(_query, project)
+    if not result:
+        raise RecordNotFoundException(identifier=query.__str__())
+    return result

core/utils/json.py

@@ -0,0 +1,42 @@
+import json
+from datetime import datetime
+from typing import Optional, Any, Union
+from uuid import UUID
+
+#
+# def uuid_encoder():
+#     print(obj)
+#     if isinstance(obj, UUID):
+#         # if the obj is uuid, we simply return the value of uuid
+#         return obj.hex
+#     return json.JSONEncoder.default(self, obj)
+#
+import jsonpatch
+from pydantic import Field
+
+from core.models.base import BaseModel
+
+
+class DataModelPatch(BaseModel):
+    id: Optional[Union[UUID, str]] = Field(alias='_id')
+    patch: Optional[Any]
+    ref_id: UUID
+    collection: str
+    owner: UUID
+    created_at: Optional[datetime]
+
+    class DB:
+        collection: str = 'revisions'
+
+
+async def set_revision(collection: str, old: str | dict, new: str | dict, ref_id: UUID, user_id: UUID):
+    old = json.dumps(old, default=str) if isinstance(old, dict) else old
+    new = json.dumps(new, default=str) if isinstance(new, dict) else new
+    patch = jsonpatch.JsonPatch.from_diff(
+        json.loads(new), json.loads(old)
+    )
+    await DataModelPatch(
+        patch=json.loads(patch.to_string()),
+        ref_id=str(ref_id),
+        collection=collection,
+        owner=str(user_id)).init_id().c_at().save()

core/utils/mongo.py

@@ -0,0 +1,47 @@
+from typing import List, Dict, Any
+
+from pymongo.collection import Collection
+
+from config import conf
+from core.exceptions import NotFoundException
+from core.utils import get_time
+
+
+def get_mongo_url(connection_name:str = None):
+    return conf.server.is_prod and conf.database.get(connection_name or 'mongo').production.url or conf.database.get(
+        connection_name or 'mongo').dev.url
+
+
+def get_mongo_db(connection_name:str = None):
+    return conf.server.is_prod and conf.database.get(connection_name or 'mongo').production.db or conf.database.get(
+        connection_name or 'mongo').dev.db
+
+
+def nested_join(collection: str, from_field, to_field,
+                alias, match: dict) -> List[Dict[str, Any]]:
+    """
+        Join a list of nested dictionaries into a single dictionary.
+    """
+    return [
+        {
+            "$lookup": {
+                "from": collection,
+                "localField": from_field,
+                "foreignField": to_field,
+                "as": alias
+            },
+        },
+        {
+            "$match": match if match else {}
+        }
+    ]
+
+
+async def use_soft_delete(collection: Collection, find_query: dict) -> bool:
+    """
+        Soft delete a document by setting a field to true.
+    """
+    result = await collection.update_one(find_query, {"$set": {'deleted_at': get_time()}, })
+    if not result.modified_count:
+        raise NotFoundException(f"No document found with query: {find_query}")
+    return True

core/utils/number.py

@@ -0,0 +1,12 @@
+import re
+
+E2P_map = {'1': '۱', '2': '۲', '3': '۳', '4': '۴', '5': '۵', '6': '۶', '7': '۷', '8': '۸', '9': '۹', '0': '۰'}
+
+E2P_map = {'1': '۱', '2': '۲', '3': '۳', '4': '۴', '5': '۵', '6': '۶', '7': '۷', '8': '۸', '9': '۹', '0': '۰'}
+
+
+# convert persian number to english number
+def convert_number_to_persian(str_in: str):
+    a = map(lambda ch: E2P_map[ch] if ch in E2P_map else ch, str_in)
+    return ''.join(list(a))
+

core/utils/response.py

@@ -0,0 +1,163 @@
+from fastapi import HTTPException
+from fastapi.encoders import jsonable_encoder
+from pydantic import ValidationError
+from starlette.responses import JSONResponse
+
+from core.exceptions import SoftDeletedException, RecordNotFoundException, ModifiedCountException, ValidationException
+
+
+class ResponseModel:
+    def __init__(self, data, code=200, message="ok"):
+        self.data = data
+        self.code = code
+        self.message = message
+
+    def __str__(self):
+        return str(self.data)
+
+    def __iter__(self):
+        return {
+            "data": self.data,
+            "code": self.code,
+            "message": self.message,
+        }
+
+
+class ResponseBadRequest(ResponseModel):
+    def __init__(self, data, message="Bad Request"):
+        super().__init__(data, 400, message)
+
+
+class ResponseValidationRequest(ResponseModel):
+    def __init__(self, data, message="Bad Request"):
+        super().__init__(data, 422, message)
+
+
+class ResponseWriteErrorRequest(ResponseModel):
+    def __init__(self, data, message="Request write error"):
+        super().__init__(data, 400, message)
+
+
+class ResponseNotFound(ResponseModel):
+    def __init__(self, data, message="Not Found"):
+        super().__init__(data, 404, message)
+
+
+class ResponseUnauthorized(ResponseModel):
+    def __init__(self, data, message="Unauthorized"):
+        super().__init__(data, 401, message)
+
+
+class ResponseInternalServerError(ResponseModel):
+    def __init__(self, data, message="Internal Server Error"):
+        super().__init__(data, 500, message)
+
+
+class ResponseOk(ResponseModel):
+    def __init__(self, data, code=200, message="ok"):
+        super().__init__(data, code, message)
+
+
+class ResponseOkUpdate(ResponseOk):
+    def __init__(self, data, message="update was successful"):
+        super().__init__(data, 200, message)
+
+
+class ResponseOkCreate(ResponseOk):
+    def __init__(self, data, message="create was successful"):
+        super().__init__(data, 201, message)
+
+
+class Res:
+
+    @staticmethod
+    def json(object, status_code):
+        return JSONResponse(content=jsonable_encoder(object), status_code=status_code)
+
+    @staticmethod
+    def create(data, message="create was successful"):
+        return Res.json(ResponseOkCreate(data, message), 201)
+
+    @staticmethod
+    def fetch(data, message="fetch was successful"):
+        return Res.json(ResponseOk(data, 200, message), 200)
+
+    @staticmethod
+    def update(data, message="update was successful"):
+        return Res.json(ResponseOkUpdate(data, message), 200)
+
+    @staticmethod
+    def not_found(data, message="Not Found"):
+        return Res.json(ResponseNotFound(data, message), 404)
+
+    @staticmethod
+    def unauthorized(data, message="Unauthorized"):
+        return Res.json(ResponseUnauthorized(data, message), 401)
+
+    @staticmethod
+    def server_error(data, message="Internal Server Error"):
+        return Res.json(ResponseInternalServerError(data, message), 500)
+
+    @staticmethod
+    def bad_request(data, message="Bad Request"):
+        return Res.json(ResponseBadRequest(data, message), 400)
+
+    @staticmethod
+    def validation_error(data, message="data validation error"):
+        return Res.json(ResponseValidationRequest(data, message), 422)
+
+    @staticmethod
+    def write_error(data, message="Request write error"):
+        return Res.json(ResponseWriteErrorRequest(data, message), 400)
+
+    @staticmethod
+    def delete_error(data, message="Request delete error"):
+        return Res.json(ResponseWriteErrorRequest(data, message), 400)
+
+
+def to_http_res(res, _type, message=None):
+    if _type == "create":
+        return Res.create(res, "create was successful")
+    elif _type == "fetch":
+        return Res.fetch(res, "fetch was successful")
+    elif _type == "update":
+        return Res.update(res, "update was successful")
+    elif _type == "delete":
+        return Res.update(res, "delete was successful")
+    elif _type == "get":
+        return Res.fetch(res, "get was successful")
+    elif _type == "upload":
+        return Res.fetch(res, "upload complete")
+    elif _type == "token":
+        return Res.fetch(res, "token was generated")
+    else:
+        return Res.fetch(res, "response ok")
+
+
+def http_exception(res):
+    _return = None
+    # if isinstance(res, ResponseBadRequest):
+    #     _return = HTTPException(status_code=res.code, detail=res.message)
+    # elif isinstance(res, ResponseNotFound):
+    #     _return = HTTPException(status_code=res.code, detail=res.message)
+    # elif isinstance(res, ResponseUnauthorized):
+    #     _return = HTTPException(status_code=res.code, detail=res.message)
+    # elif isinstance(res, ResponseInternalServerError):
+    #     _return = HTTPException(status_code=res.code, detail=res.message)
+    if isinstance(res, SoftDeletedException):
+        _return = Res.delete_error(res, res.message)
+    elif isinstance(res, RecordNotFoundException):
+        _return = Res.not_found(res, res.message)
+    elif isinstance(res, ModifiedCountException):
+        _return = Res.write_error(res, res.message)
+    elif isinstance(res, ValidationError):
+        _return = Res.validation_error(res.errors())
+    elif isinstance(res, AssertionError):
+        _return = Res.validation_error(res, message=res)
+    elif isinstance(res, ValidationException):
+        _return = Res.validation_error(res.message, message=res.message)
+    elif isinstance(res, HTTPException):
+        _return = Res.bad_request(res.status_code, message=res.detail)
+    else:
+        _return = Res.server_error(res, f'unknown error: {str(res)} {type(res)}')
+    return _return

core/utils/router.py

@@ -0,0 +1,206 @@
+import time
+from functools import wraps
+from typing import Callable, TypeVar, Generic
+
+from fastapi import APIRouter, Request, Response, Body
+from fastapi import Security
+from fastapi.routing import APIRoute
+from starlette.background import BackgroundTasks
+
+from auth.utils import get_current_active_user
+from config import conf
+from core.utils import to_http_res, http_exception
+from core.utils.auth import get_current_active_user
+
+
+class BaseRoute(APIRoute):
+    def get_route_handler(self) -> Callable:
+        original_route_handler = super().get_route_handler()
+
+        async def custom_route_handler(request: Request) -> Response:
+            before = time.time()
+            response: Response = await original_route_handler(request)
+            duration = time.time() - before
+            response.headers["X-Response-Time"] = str(duration)
+            return response
+
+        return custom_route_handler
+
+
+def shrink(cal: Callable, name: str = None, bt: BackgroundTasks = None):
+    def can_inner(func):
+        @wraps(func)
+        async def wrapper(*args, **kwargs):
+            return await safe_route(name or cal.__name__, cal, *kwargs.values())
+        return wrapper
+
+    return can_inner
+
+
+def back_task(cal: Callable, name: str = None):
+    print(cal)
+
+    def back_inner(func):
+        print(func)
+
+        @wraps(func)
+        async def wrapper(*args, **kwargs):
+            if cal.__name__ == 'create_salam':
+                print(cal.__name__)
+            return await safe_route(name or cal.__name__, cal, *kwargs.values())
+
+        return wrapper
+
+    return back_inner
+
+
+def generate_router(
+        prefix,
+        dependencies=None,
+        auth_dependencies=False,
+        version='v1',
+        auth_scopes=['account']):
+    """
+    Generate a router with a prefix and dependencies.
+    Args:
+        prefix: prefix of the router.
+        dependencies: dependencies of the router.
+        auth_dependencies:  if router has auth dependencies is True.
+        version: version of the router.
+        auth_scopes: auth scopes of the router.
+
+    Returns: ApiRouter
+    """
+    if auth_dependencies:
+        dependencies = dependencies or []
+        if 'customer' in auth_scopes:
+            dependencies.extend(
+                [Security(get_current_active_customer, scopes=auth_scopes)])
+        else:
+            dependencies.extend(
+                [Security(get_current_active_user, scopes=auth_scopes)])
+    _pre = conf.server.prefix_url.replace('v1', version)
+    return APIRouter(
+        prefix=f"{_pre}/{prefix}",
+        route_class=BaseRoute,
+        dependencies=dependencies if dependencies else []
+    )
+
+
+async def safe_route(_type, fn, *args):
+    try:
+        return to_http_res(await fn(*args), _type)
+    except Exception as e:
+        return http_exception(e)
+
+
+class CrudMethods:
+    create = True
+    read = True
+    update = True
+    delete = True
+    get_all = True
+
+    def __init__(self, create=True, read=True, update=True, delete=True, fetch=True):
+        self.create = create
+        self.read = read
+        self.update = update
+        self.delete = delete
+        self.fetch = fetch
+
+    def __iter__(self):
+        r = []
+        if self.create:
+            r.append('create')
+        if self.read:
+            r.append('read')
+        if self.update:
+            r.append('update')
+        if self.delete:
+            r.append('delete')
+        if self.fetch:
+            r.append('fetch')
+        return r
+
+
+T = TypeVar("T")
+
+
+class SetupCrud(Generic[T]):
+    _url_prefix = {
+        'create': '/create',
+        'get': '/{_id}',
+        'update': '/{_id}',
+        'delete': '/{_id}',
+        'fetch': ''
+    }
+    _methods = {
+        'create': 'post',
+        'fetch': 'post',
+        'get': 'get',
+        'update': 'put',
+        'delete': 'delete',
+    }
+
+    def __init__(self,
+                 router: APIRouter,
+                 repository,
+                 model: T,
+                 tags=[],
+                 methods: CrudMethods = CrudMethods()
+                 ):
+
+        for i in ['create', 'update', 'get', 'delete', 'fetch']:
+            if i == 'create':
+                @router.post(
+                    self._url_prefix[i],
+                    response_model=model,
+                    tags=tags
+                )
+                async def create(body=Body(...)):
+                    b = T(**body)
+                    return await safe_route(i, getattr(repository, 'create'), b)
+            # elif i == 'update':
+            #     async def update(_id, body: T = Body(...)):
+            #         return await safe_route(i, getattr(repository, i), _id, body)
+            #
+            #     router.add_api_route(
+            #         _url_prefix[i],
+            #         update,
+            #         methods=[_methods[i]],
+            #         response_model=model,
+            #         tags=tags
+            #     )
+            # elif i == 'delete':
+            #     async def delete(_id):
+            #         return await safe_route(i, getattr(repository, i), _id)
+            #
+            #     router.add_api_route(
+            #         _url_prefix[i],
+            #         delete,
+            #         methods=[_methods[i]],
+            #         response_model=model,
+            #         tags=tags
+            #     )
+            # elif i == 'get':
+            #     async def get(_id):
+            #         return await safe_route(i, getattr(repository, i), _id)
+            #
+            #     router.add_api_route(
+            #         _url_prefix[i],
+            #         get,
+            #         methods=[_methods[i]],
+            #         response_model=model,
+            #         tags=tags
+            #     )
+            # elif i == 'fetch':
+            #     async def fetch(_id, body: T = Body(...)):
+            #         return await safe_route(i, getattr(repository, i), _id, body)
+            #
+            #     router.add_api_route(
+            #         _url_prefix[i],
+            #         fetch,
+            #         methods=[_methods[i]],
+            #         response_model=model,
+            #         tags=tags
+            #     )

core/utils/sumery/__init__.py

@@ -0,0 +1,89 @@
+"""
+smart upon mini query
+amazing mini query in any database
+"""
+
+from core.models.base import DELETED_AT_QUERY
+from core.utils import smart_cast
+from core.service.mongo_db import use_collection
+from core.exceptions import ValidationException
+
+
+class SumeryObject:
+    db: str
+    object_name: str
+    table_name: str
+    driver: str = 'mongo'
+    query_fn: callable
+    cols_search: list
+    key_col: str = '_id'
+    value_col: str
+    value_fn: callable
+    limit: 10
+
+    def __init__(self, db: str,
+                 object_name: str,
+                 cols_search: list,
+                 key_col: str,
+                 value_col: str,
+                 driver: str = 'mongo',
+                 table_name: str = None,
+                 query_fn: callable = None,
+                 limit: int = 10,
+                 value_fn: callable = None,
+                 ):
+        self.db = db
+        self.table_name = table_name
+        self.object_name = object_name
+        self.driver = driver
+        self.query_fn = query_fn
+        self.cols_search = cols_search
+        self.key_col = key_col
+        self.value_col = value_col
+        self.value_fn = value_fn
+        self.limit = limit
+
+
+__sumery_list: list[SumeryObject] = []
+
+
+async def sumery_executor(object_name: str, query: str, extra_filter: dict = None):
+    __query = query.strip()
+    if len(__query) < 3:
+        raise ValidationException('length of query not valid')
+    _so = next((x for x in __sumery_list if x.object_name == object_name), None)
+    if not _so:
+        raise ValidationException('summery object not found')
+    _query = {'$and': [{'$or': [{x: {'$regex': __query}} for x in _so.cols_search]},
+                       DELETED_AT_QUERY,
+                       extra_filter if extra_filter else {}]}
+    _projection = {
+        '_id': False,
+        'key': f'${_so.key_col}',
+        'value': f'${_so.value_col}'
+    }
+    records = use_collection(_so.table_name or _so.object_name, _so.db).find(_query, _projection).limit(_so.limit)
+    return [x async for x in records]
+
+
+async def sumery_default_getter(object_name: str, ids: list, extra_filter: dict = None):
+    if not len(ids):
+        raise ValidationException('length of ids not valid')
+    _so = next((x for x in __sumery_list if x.object_name == object_name), None)
+    if not _so:
+        raise ValidationException('summery object not found')
+    __cast_ids = [smart_cast(x) for x in ids]
+    _query = {'$and': [{_so.key_col: {'$in': __cast_ids}},
+                       DELETED_AT_QUERY,
+                       extra_filter if extra_filter else {}]}
+    _projection = {
+        '_id': False,
+        'key': f'${_so.key_col}',
+        'value': f'${_so.value_col}'
+    }
+    records = use_collection(_so.table_name or _so.object_name, _so.db).find(_query, _projection).limit(100)
+    return [x async for x in records]
+
+
+def setup_sumery_object(so: SumeryObject):
+    __sumery_list.append(so)

core/utils/table_engine/__init__.py

@@ -0,0 +1,4 @@
+from .engine import *
+from .filter import *
+from .input import *
+from .types import *

core/utils/table_engine/engine.py

@@ -0,0 +1,112 @@
+from pydantic import BaseModel
+from sqlalchemy import func
+
+from core.models.base import ORDER_BY_CREATED_AT
+from core.utils.table_engine.filter import TableEngineFilter
+from core.utils.table_engine.input import TableEngineInput
+
+
+class TableEngine:
+    object: any
+    input: TableEngineInput
+    is_sql_server: bool
+    is_deleted: bool
+    model: any
+    joins: list = None
+    filter: TableEngineFilter
+    sort: dict = ORDER_BY_CREATED_AT
+    extra_filter: list
+
+    def __init__(self, _object, _input: dict):
+        self.object = _object
+        _input = _input or {}
+        self.input = TableEngineInput(**_input)
+        self.input.get_limit()
+        self.filter = TableEngineFilter(self.input)
+        self.extra_filter = []
+
+    def soft_deleted(self):
+        self.is_deleted = True
+        self.input.is_deleted = True
+        return self
+
+    def sql_server(self):
+        self.is_sql_server = True
+        return self
+
+    def text_search_cols(self, cols: list):
+        self.input.text_search_cols(cols)
+        return self
+
+    def set_limit(self, limit: int):
+        self.input.p.s = limit
+        return self
+
+    def set_model(self, model: BaseModel.__class__):
+        self.model = model
+        return self
+
+    def join(self, joins):
+        self.joins = joins
+        return self
+
+    def sort_by(self, sort_by: dict | str):
+        self.sort = sort_by
+        return self
+
+    def _get_sql_order(self):
+        if isinstance(self.sort, dict):
+            return self.model.Id.desc()
+        return self.sort
+
+    def add_filter(self, filters):
+        self.extra_filter.append(filters)
+        self.input.extra_filter.append(filters)
+        return self
+
+    def get_query(self):
+        return self.input.dict()
+
+    async def exec(self):
+        if hasattr(self, 'is_sql_server') and self.is_sql_server:
+            return await self._exec_sql_server()
+        else:
+            return await self._exec_mongo()
+
+    async def _exec_mongo(self):
+        cursor = self.object.find(
+            self.filter.get()).sort(
+            **self.sort).limit(
+            self.input.get_limit()).skip(self.input.skip())
+        return [self.model(**document) async for document in cursor]
+
+    async def _exec_sql_server(self):
+        query = self.object \
+            .query(self.model)
+
+        if self.joins:
+            for j in self.joins:
+                if isinstance(j, list):
+                    query = query.join(*j)
+                else:
+                    query = query.join(j)
+
+        query = self.filter.get_sql_server(
+            self.model, query, self.extra_filter)
+
+        count = query.count()
+
+        query = query \
+            .order_by(self._get_sql_order()) \
+            .offset(self.input.skip()) \
+            .limit(self.input.get_limit())
+
+        return query.all(), count
+
+    async def count(self):
+        if hasattr(self, 'is_sql_server') and self.is_sql_server:
+            count = self.object.query(func.count(self.model.Id))
+        else:
+            count = await self.object.count_documents(
+                self.filter.get())
+        return count

core/utils/table_engine/filter.py

@@ -0,0 +1,160 @@
+__all__ = ['TableEngineFilter']
+
+from uuid import UUID
+
+from sqlalchemy import or_
+
+from core.utils.table_engine.input import TableEngineInput, FilterItem
+from core.utils.utils import is_valid_uuid
+
+
+def filter_item_generator(item: FilterItem) -> dict:
+    if is_valid_uuid(item.col):
+        return {
+            'data': {
+                "$elemMatch": {
+                    'field_id': UUID(item.col),
+                    'value': {**get_expression_filter(item.op, cast_value(item.val, item.type))}
+                }
+            }
+        }
+
+    return {item.col: {
+        **get_expression_filter(item.op, cast_value(item.val, item.type))}}
+
+
+def cast_value(val: str, type: str) -> any:
+    if type in ['NUMBER', 'STACK', 'FLOAT']:
+        return float(val)
+    if type == 'UUID':
+        return UUID(str(val))
+    return val
+
+
+def get_expression_filter(op: str, val: str) -> dict:
+    _op = get_operator(op)
+    match _op:
+        case "sw":
+            return {'$regex': f'^{val}'}
+        case "ew":
+            return {'$regex': f'{val}$'}
+        case "n_ew":
+            return {'$regex': f'(?<!{val})$'}
+        case "n_sw":
+            return {'$regex': f'^(?!{val})'}
+        case "n_like":
+            return {'$not': {'$regex': val}}
+        case "bet":
+            if type(val) is str:
+                return {'$gte': val.split('-')[0], '$lte': val.split('-')[1]}
+            return {'$gte': val[0], '$lte': val[1]}
+        case "n_bet":
+            if type(val) is str:
+                return {'$gte': val.split('-')[0], '$lte': val.split('-')[1]}
+            return {'$not': {'$gte': val[0], '$lte': val[1]}}
+        case "exi":
+            return {'$exists': True}
+        case "n_exi":
+            return {'$exists': False}
+        case "in":
+            return {'$exists': True}
+        case "not_in":
+            return {'$exists': False}
+    return {_op: val}
+
+
+def get_operator(op: str) -> str:
+    ops = {
+        "eq": "$eq",
+        "n_eq": "$ne",
+        "gt": "$gt",
+        "ge": "$ge",
+        "lt": "$lt",
+        "le": "$le",
+        "like": "$regex",
+        "n_like": "n_like",
+        "sw": "sw",
+        "ew": "ew",
+        'n_ew': 'n_ew',
+        'n_sw': 'n_sw',
+        'exi': 'exi',
+        "n_exi": "n_exi",
+        'lte': '$lte',
+        'gte': '$gte',
+        'in': 'in',
+        'n_in': 'not_in',
+        'bet': 'bet',
+        'n_bet': 'n_bet',
+    }
+
+    return ops[op] if op in ops else '$eq'
+
+
+def getAndOr(op: str) -> str:
+    return {"and": "$and", "or": '$or'}.get(op, '$and')
+
+
+class TableEngineFilter:
+    stack: list
+    input: TableEngineInput
+
+    def __init__(self, _input: TableEngineInput):
+        self.stack = []
+        self.input = _input
+
+    def add(self, filter_item: dict):
+        self.stack.append(filter_item)
+
+    def query_generator(self):
+        if not self.input.has_filter():
+            return []
+
+        filters = self.input.filters()
+        query = []
+        for filter_scope in filters:
+            query.append({getAndOr(filter_scope.op): [
+                filter_item_generator(item) for item in filter_scope.items]})
+        return query
+
+    def get(self):
+        f = self.stack
+        query = self.query_generator()
+        if query:
+            f.append(*query)
+        if self.input.has_text_search():
+            items = {}
+            for col in self.input.get_search_cols():
+                items[col] = {'$regex': self.input.get_text_search()}
+            f.append(items)
+        if hasattr(self.input, 'is_deleted') and self.input.is_deleted:
+            f.append({'deleted_at': {'$exists': False}})
+        if len(self.input.extra_filter) > 0:
+            f.extend(self.input.extra_filter)
+        if not f:
+            return {}
+        return {'$and': f}
+
+    def get_sql_server(self, model, query, filters):
+        """
+         text search filter
+        """
+        text_search_stack = []
+        fi = []
+        if self.input.has_text_search():
+            for col in self.input.get_search_cols():
+                if isinstance(col, str):
+                    text_search_stack.append(getattr(model, col).like(
+                        f'%{self.input.get_text_search()}%'))
+
+            if text_search_stack:
+                fi.append(or_(*text_search_stack))
+
+        if self.input.is_deleted:
+            fi.append(getattr(model, 'IsDeleted') == 0)
+
+        if filters:
+            for fil in filters:
+                fi.append(fil)
+
+        query = query.filter(*fi)
+        return query

core/utils/table_engine/input.py

@@ -0,0 +1,159 @@
+from enum import Enum
+from typing import List, Optional, Any
+
+import pydantic
+from pydantic import BaseModel
+
+
+class Cr(str, Enum):
+    OR = 'or'
+    AND = 'and'
+
+
+class FilterItem(BaseModel):
+    col: Optional[str]
+    cr: Optional[Cr]
+    id: Optional[str]
+    op: Optional[str]
+    type: Optional[str]
+    val: Optional[Any]
+
+    @staticmethod
+    def ValidItem(item: dict):
+        return item.get('col') and \
+            item.get('id') and \
+            item.get('op') and \
+            item.get('type')
+
+    @staticmethod
+    def ValidValue(item: dict):
+        return len(str(item.get('val', '')).strip()) > 0
+
+
+class TableEngineInput(BaseModel):
+
+    class FilterScope(BaseModel):
+        id: str
+        items: Optional[List[FilterItem]]
+        op: Optional[Cr]
+
+        def __init__(self, **kwargs):
+            super().__init__(**kwargs)
+            if kwargs.get('items'):
+                self.items = [
+                    FilterItem(
+                        **item) for item in filter(
+                        lambda i: FilterItem.ValidItem(i),
+                        kwargs.get(
+                            'items',
+                            []))]
+            self.op = Cr(kwargs.get('op'))
+            self.id = str(kwargs.get('id'))
+
+        @staticmethod
+        def ValidScope(item: dict):
+            return item.get('id') and item.get('items') and len(
+                item.get('items')) > 0 and item.get('op')
+
+        @staticmethod
+        def CleanItems(item):
+            clone = item.dict()
+            clone['items'] = list(filter(
+                lambda i: FilterItem.ValidValue(i), clone.get(
+                    'items', [])))
+            return TableEngineInput.FilterScope(**clone)
+
+    class Page(BaseModel):
+        c: int = 1
+        s: int = 5
+
+    class SearchModel(BaseModel):
+        text: Optional[str]
+        cols: Optional[List[Any]]
+
+    class SortModel(BaseModel):
+        field: Optional[str]
+        direction: Optional[str]
+
+    class Config:
+        extra = pydantic.Extra.ignore
+        anystr_strip_whitespace = True  # strip whitespaces from strings
+        underscore_attrs_are_private = False
+        arbitrary_types_allowed = True
+
+    f: Optional[List[FilterScope]]
+    extra_filter: list = []
+    o: Optional[SortModel]
+    p: Optional[Page] = Page()
+    s: Optional[SearchModel]
+    is_deleted: Optional[bool]
+
+    def limit(self):
+        return self.p.s
+
+    def order(self):
+        return self.p.o
+
+    def filters(self) -> List[FilterScope]:
+        return self.f
+
+    def has_filter(self) -> bool:
+        return self.f is not None and \
+            len(self.f) > 0 and \
+            self.f[0].items is not None and \
+            len(self.f[0].items) > 0
+
+    def has_extra_filter(self) -> bool:
+        return self.extra_filter is not None and \
+            len(self.extra_filter) > 0
+
+    def scopes(self):
+        return self.s
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        _input = {**{'p': {'c': 1, 's': 5},
+                     'f': [], 's': {}, 'o': {}}, **(kwargs or {})}
+        self.p = TableEngineInput.Page(**_input.get('p'))
+
+        if _input.get('f'):
+            self.f = [
+                TableEngineInput.FilterScope(
+                    **f) for f in filter(
+                    lambda fs:TableEngineInput.FilterScope.ValidScope(fs),
+                    _input.get('f'))]
+            self.f = list(map(
+                lambda fs: TableEngineInput.FilterScope.CleanItems(fs),
+                self.f))
+
+        if _input.get('s'):
+            self.s = TableEngineInput.SearchModel(**_input.get('s'))
+
+    def skip(self):
+        return self.page() * self.p.s
+
+    def page(self):
+        return self.p.c - 1
+
+    def virtual_page(self):
+        return self.page() + 1
+
+    def get_limit(self):
+        return self.p.s if self.p else 20
+
+    def text_search_cols(self, cols: List[str]):
+        if self.s:
+            self.s.cols = cols
+
+    def has_text_search(self) -> bool:
+        return self.s is not None and self.s.text is not None and self.s.cols is not None and len(
+            self.s.cols) > 0 and len(self.s.text.strip()) > 0
+
+    def add_filter(self, _filter):
+        self.extra_filter.append(_filter)
+
+    def get_search_cols(self) -> List[str]:
+        return self.s.cols if self.s else []
+
+    def get_text_search(self) -> str:
+        return self.s.text.strip() if self.s else None

core/utils/table_engine/types.py

@@ -0,0 +1,23 @@
+from typing import Generic, TypeVar, List, Optional
+
+from pydantic.generics import GenericModel
+
+T = TypeVar('T')
+
+
+class TableEngineInputType(GenericModel, Generic[T]):
+    f: Optional[List]
+    o: Optional[dict] = {}
+    p: Optional[dict] = {}
+    s: Optional[dict] = {}
+
+
+class TableEngineExport(GenericModel, Generic[T]):
+    rows: Optional[List[T]]
+    total: Optional[int]
+    page: Optional[int]
+    te: Optional[TableEngineInputType]
+
+
+class TableEngineRequestType(GenericModel, Generic[T]):
+    te: Optional[TableEngineInputType]

core/utils/te/__init__.py

@@ -0,0 +1,77 @@
+from typing import List
+
+from .engines import MsSqlEngine, MongoEngine, PgSqlEngine, AbstractEngine
+from .utils.Models import TableEngineInput
+from .utils.Models import WorkingMode, JoinModel
+from ..table_engine import TableEngineExport
+
+
+class NewTableEngine:
+    def __init__(self, query_class, model, inputs, mode, engine: AbstractEngine, extra_filter,
+                 default_sort_by, joins, search_cols, options=None, soft_deleted=False):
+        self.model = model
+        self.query_class = query_class
+        self.mode = mode
+        self.joins = False
+        self.raw_input = inputs
+        if isinstance(inputs, dict):
+            self.input = TableEngineInput(**inputs if inputs else {})
+        else:
+            self.input = TableEngineInput(**inputs.dict() if inputs else {})
+        self.input.extra_filter = extra_filter
+        self.default_sort_by = default_sort_by
+        self.joins = joins
+        self.soft_deleted = soft_deleted
+        self.options = options
+        self.search_cols = search_cols
+        self.engine = engine(self)
+
+    @staticmethod
+    def mongo(query_class,
+              model,
+              inputs,
+              mode: WorkingMode = WorkingMode.Development,
+              extra_filter=None,
+              soft_deleted=False,
+              default_sort_by=None, joins=None, search_cols=[]):
+        return NewTableEngine(query_class, model, inputs, mode, engine=MongoEngine, extra_filter=extra_filter,
+                              default_sort_by=default_sort_by, joins=joins, search_cols=search_cols,
+                              soft_deleted=soft_deleted)
+
+    @staticmethod
+    def mssql(query_class, model, inputs, mode: WorkingMode = WorkingMode.Development, extra_filter=None,
+              default_sort_by=None, joins=None, search_cols=[], options=None):
+        return NewTableEngine(query_class, model, inputs, mode, engine=MsSqlEngine, extra_filter=extra_filter,
+                              default_sort_by=default_sort_by, joins=joins, search_cols=search_cols, options=options)
+
+    @staticmethod
+    def pgsql(query_class, model, inputs, mode: WorkingMode = WorkingMode.Development, extra_filter=None,
+              default_sort_by=None, joins=None, search_cols=None, options=None):
+        return NewTableEngine(query_class, model, inputs, mode, engine=PgSqlEngine, extra_filter=extra_filter,
+                              default_sort_by=default_sort_by, joins=joins, search_cols=search_cols, options=options)
+
+    async def exe(self):
+        return await self.engine.exe()
+
+    async def to_dict(self) -> TableEngineExport:
+        rows, count = await self.exe()
+        self.input.extra_filter = None
+        del self.engine
+        return TableEngineExport(
+            rows=rows,
+            page=self.input.virtual_page(),
+            total=count,
+            te=self.input
+        )
+
+    def add_extra_filter(self, filters):
+        self.input = filters
+        return self
+
+    def joins(self, joins: List[JoinModel]):
+        self.joins = joins
+        return self
+
+    def set_custom_entities(self, entities=None):
+        self.engine.custom_entities = entities
+        return self

core/utils/te/engines/AbstractEngine.py

@@ -0,0 +1,51 @@
+import abc
+
+__all__ = "AbstractEngine"
+
+
+class AbstractEngineClass(abc.ABC):
+    def __init__(self, table_engine):
+        self.model = table_engine.model
+        self.query_class = table_engine.query_class
+        self.input = table_engine.input
+        self.filter = table_engine.input.f
+        self.default_sort_by = table_engine.default_sort_by
+        self.custom_entities = []
+        self.joins = table_engine.joins
+        self.options = table_engine.options
+        self.search_cols = table_engine.search_cols
+        self.soft_deleted = table_engine.soft_deleted
+
+    @abc.abstractmethod
+    def exe(self):
+        pass
+
+    @abc.abstractmethod
+    def set_filters(self):
+        pass
+
+    @abc.abstractmethod
+    def set_sort(self):
+        pass
+
+    @staticmethod
+    @abc.abstractmethod
+    def get_expression_filter(col=None, op: str = 'eq', val: str = '') -> dict:
+        pass
+
+    @staticmethod
+    @abc.abstractmethod
+    def get_operator(op: str):
+        pass
+
+    def count(self):
+        pass
+
+    @staticmethod
+    @abc.abstractmethod
+    def generate_scope_filter(filters: list, general=False):
+        pass
+
+    @abc.abstractmethod
+    def set_joins(self):
+        pass

core/utils/te/engines/Mongo.py

@@ -0,0 +1,187 @@
+import datetime
+from typing import Union, List
+from uuid import UUID
+
+from .AbstractEngine import AbstractEngineClass
+from ..utils.Models import FilterItem, FilterScope
+
+
+class MongoEngine(AbstractEngineClass):
+    rows = []
+    stack = []
+    filters = {}
+
+    async def exe(self):
+        _f = self.filters = self.set_filters()
+        tmp = self.query_class.find(_f).sort(**self.set_sort())
+        self.rows = tmp.limit(
+            self.input.get_limit()).skip(self.input.skip())
+        return [self.model(**document) async for document in self.rows], await self.count()
+
+    def query_generator(self):
+        if not self.input.has_filter():
+            return []
+        filters = self.input.filters()
+        query = self.generate_scope_filter(filters, general=True)
+        return query
+
+    def set_filters(self):
+        filters = []
+        query = self.query_generator()
+        if query:
+            filters.append(query)
+        if self.input.has_text_search():
+            items = []
+            _search_cols = self.input.get_search_cols() if (
+                    self.input.get_search_cols() and len(self.input.get_search_cols())) else self.search_cols
+            if _search_cols:
+                for col in _search_cols:
+                    items.append({col: {'$regex': self.input.get_text_search()}})
+                filters.append({"$or": items})
+        if self.input.extra_filter:
+            filters.extend(self.input.extra_filter)
+        if self.soft_deleted:
+            filters.append({'deleted_at': {'$exists': False}})
+        if not filters:
+            return {}
+        self.filters = {'$and': filters}
+        return self.filters
+
+    def set_sort(self):
+        if self.input.o.field:
+            return {'key_or_list': self.input.o.field,
+                    'direction': 1 if self.input.o.direction == "asc" else -1}
+        elif self.default_sort_by:
+            return self.default_sort_by
+        else:
+            return {'key_or_list': 'created_at', 'direction': -1}
+
+    @staticmethod
+    def filter_item_generator(item: FilterItem) -> dict:
+        if MongoEngine.is_valid_uuid(str(item.col)):
+            return {
+                'data': {
+                    "$elemMatch": {
+                        'field_id': UUID(item.col),
+                        'value': {
+                            **MongoEngine.get_expression_filter(item.op, MongoEngine.cast_value(item.val, item.type), )}
+                    }
+                }
+            }
+        return {item.col: {
+            **MongoEngine.get_expression_filter(col=item.col, op=item.op,
+                                                val=MongoEngine.cast_value(item.val, item.type), )}}
+
+    @staticmethod
+    def is_valid_uuid(uuid_to_test, version=4):
+        try:
+            uuid_obj = UUID(str(uuid_to_test), version=version)
+        except ValueError:
+            return False
+        return str(uuid_obj) == str(uuid_to_test)
+
+
+    @staticmethod
+    def cast_value(val: Union[str, List], type: str) -> any:
+        if type in ['NUMBER', 'STACK', 'FLOAT']:
+            if isinstance(val, List):
+                return [float(x) for x in val]
+            return float(val)
+        if type in ['DATE', 'DATETIME']:
+            if isinstance(val, List):
+                return [datetime.datetime.fromisoformat(str(x)) for x in val]
+            return datetime.datetime.fromisoformat(val)
+        if type == 'UUID':
+            if isinstance(val, List):
+                return [UUID(str(x)) for x in val]
+            return UUID(str(val))
+        return val
+
+
+    @staticmethod
+    def get_expression_filter(col=None, op: str = 'eq', val: str = '') -> dict:
+        _op = MongoEngine.get_operator(op)
+        if _op == "sw":
+            return {'$regex': f'^{val}'}
+        elif _op == "ew":
+            return {'$regex': f'{val}$'}
+        elif _op == "n_ew":
+            return {'$regex': f'(?<!{val})$'}
+        elif _op == "n_sw":
+            return {'$regex': f'^(?!{val})'}
+        elif _op == "n_like":
+            return {'$not': {'$regex': val}}
+        elif _op == "bet":
+            if isinstance(val, str):
+                return {'$gte': val.split('-')[0], '$lte': val.split('-')[1]}
+            return {'$gte': val[0], '$lte': val[1]}
+        elif _op == "n_bet":
+            if isinstance(val, str):
+                return {'$gte': val.split('-')[0], '$lte': val.split('-')[1]}
+            return {'$not': {'$gte': val[0], '$lte': val[1]}}
+        elif _op == "exi":
+            return {'$exists': True}
+        elif _op == "n_exi":
+            return {'$exists': False}
+        elif _op == "in":
+            return {'$exists': True}
+        elif _op == "not_in":
+            return {'$exists': False}
+        return {_op: val}
+
+    @staticmethod
+    def get_operator(op: str) -> str:
+        ops = {
+            "eq": "$eq",
+            "n_eq": "$ne",
+            "gt": "$gt",
+            "ge": "$ge",
+            "lt": "$lt",
+            "le": "$le",
+            "like": "$regex",
+            "n_like": "n_like",
+            "sw": "sw",
+            "ew": "ew",
+            'n_ew': 'n_ew',
+            'n_sw': 'n_sw',
+            'exi': 'exi',
+            "n_exi": "n_exi",
+            'lte': '$lte',
+            'gte': '$gte',
+            'in': 'in',
+            'n_in': 'not_in',
+            'bet': 'bet',
+            'n_bet': 'n_bet',
+        }
+        return ops[op] if op in ops else '$eq'
+
+    async def count(self):
+        return await self.query_class.count_documents(self.filters)
+
+    @staticmethod
+    def generate_scope_filter(filters: list, general=False):
+        if len(filters) == 1:
+            _o = filters[0]
+            if isinstance(_o, FilterItem):
+                return MongoEngine.filter_item_generator(filters[0])
+            elif isinstance(_o, FilterScope):
+                return MongoEngine.generate_scope_filter(_o.items)
+            else:
+                return _o
+        else:
+            _a = filters.pop(0)
+            _a = (MongoEngine.filter_item_generator(_a) if isinstance(_a,
+                                                                      FilterItem) else _a) if not general else (
+                MongoEngine.generate_scope_filter(_a.items) if isinstance(_a, FilterScope) else _a)
+            _b = filters.pop(0)
+            op = '$' + (_b.cr if not general else _b.op)
+            _b = (MongoEngine.filter_item_generator(_b) if isinstance(_b,
+                                                                      FilterItem) else _b) if not general else (
+                MongoEngine.generate_scope_filter(_b.items) if isinstance(_b, FilterScope) else _b)
+            filters = [{op: [
+                _a, _b
+            ]}] + filters
+            return MongoEngine.generate_scope_filter(filters, general)
+
+    def set_joins(self):
+        pass

core/utils/te/engines/Mssql.py

@@ -0,0 +1,146 @@
+from sqlalchemy import or_, and_, desc, asc
+from sqlalchemy.sql.elements import BooleanClauseList, BinaryExpression, UnaryExpression
+
+from .AbstractEngine import AbstractEngineClass
+from ..utils.Models import FilterItem, FilterScope
+
+
+class MsSqlEngine(AbstractEngineClass):
+
+    def set_joins(self):
+        pass
+
+    filters = {}
+    rows = []
+    sort = None
+
+    async def exe(self):
+        _f = self.set_filters()
+        if not self.custom_entities:
+            query = self.query_class.query(self.model)
+        else:
+            query = self.query_class.query(*self.custom_entities)
+        if self.joins:
+            for j in self.joins:
+                if isinstance(j, list):
+                    query = query.join(*j)
+                else:
+                    query = query.join(j)
+
+        if self.options:
+            query = query.options(*self.options)
+        tmp = query.filter(_f).order_by(self.set_sort())
+        self.rows = tmp.limit(self.input.get_limit()).offset(self.input.skip()).all()
+        return self.rows, tmp.count(), tmp
+
+    def query_generator(self):
+        if not self.input.has_filter():
+            return None
+        filters = self.input.filters()
+        query = self.generate_scope_filter(filters, general=True)
+        return query
+
+    def set_sort(self):
+        if self.input.o.field:
+            self.sort = asc(getattr(self.model, self.input.o.field)) if self.input.o.direction == 'asc' else desc(
+                getattr(self.model, self.input.o.field))
+            return self.sort
+        elif isinstance(self.default_sort_by, UnaryExpression):
+            return self.default_sort_by
+        else:
+            return True
+
+    def set_filters(self):
+        filters = []
+        query = self.query_generator()
+        if isinstance(query, BooleanClauseList) or isinstance(query, BinaryExpression):
+            filters.append(query)
+        if self.input.has_text_search():
+            items = []
+            _search_cols = self.input.get_search_cols() if (
+                    self.input.get_search_cols() and len(self.input.get_search_cols())) else self.search_cols
+            if _search_cols:
+                for col in _search_cols:
+                    has_join = len(col.split('.')) > 1
+                    if has_join:
+                        pass
+                    else:
+                        items.append(
+                            getattr(self.model, col).ilike("%{search}%".format(search=self.input.get_text_search())))
+                filters.append(or_(*items))
+        if self.input.extra_filter:
+            for f in self.input.extra_filter:
+                filters.append(f)
+        if not filters:
+            return True
+        self.filters = and_(*filters)
+        return self.filters
+
+    @staticmethod
+    def get_expression_filter(col=None, op: str = '', val: str = '') -> dict:
+        _op = MsSqlEngine.get_operator(op)
+        return getattr(col, op)(val)
+
+    @staticmethod
+    def get_operator(op: str):
+        ops = {
+            "eq": "__eq__",
+            "n_eq": "__ne__",
+            "gt": "__gt__",
+            "gte": "__ge__",
+            "lte": "__lte__",
+            "lt": "__lt__",
+            "like": "ilike",
+            "n_like": "notlike",
+            "in": "in_",
+            "n_in": "not_in",
+            "bet": "between",
+            "n_bet": "not_between",
+            "sw": "startswith",
+            "ew": "endswith",
+            'n_ew': 'not_endswith',
+            'n_sw': 'not_startswith',
+            'exi': 'exi',
+            "n_exi": "n_exi",
+        }
+        return ops[op] if op in ops else '__eq__'
+
+    @staticmethod
+    def get_and_or(op):
+        return or_ if op == 'or' else and_
+
+    def count(self):
+        pass
+
+    def filter_item_generator(self, item: FilterItem):
+        has_join = len(item.col.split('.')) > 1
+        if has_join:
+
+            pass
+        else:
+            return getattr(getattr(self.model, item.col), self.get_operator(item.op))(item.val)
+
+    def generate_scope_filter(self, filters: list, general=False):
+        if len(filters) == 1:
+            _o = filters[0]
+            if isinstance(_o, FilterItem):
+                return self.filter_item_generator(filters[0])
+            elif isinstance(_o, FilterScope):
+                return self.generate_scope_filter(_o.items)
+            else:
+                return _o
+        else:
+            _a = filters.pop(0)
+            _a = (self.filter_item_generator(_a) if isinstance(_a,
+                                                               FilterItem) else _a) if not general else (
+                self.generate_scope_filter(_a.items) if isinstance(_a, FilterScope) else _a)
+
+            _b = filters.pop(0)
+
+            _op = MsSqlEngine.get_and_or(_b.cr if not general else _b.op)
+            _b = (self.filter_item_generator(_b) if isinstance(_b,
+                                                               FilterItem) else _b) if not general else (
+                self.generate_scope_filter(_b.items) if isinstance(_b, FilterScope) else _b)
+
+            filters = [_op(_a, _b)] + filters
+            return self.generate_scope_filter(filters, general)

core/utils/te/engines/Pgsql.py

@@ -0,0 +1,13 @@
+from .AbstractEngine import AbstractEngineClass
+
+
+class PgSqlEngine(AbstractEngineClass):
+
+    def exe(self):
+        pass
+
+    def get_filters(self):
+        pass
+
+    def pagination(self):
+        pass

core/utils/te/engines/__init__.py

@@ -0,0 +1,4 @@
+from enum import Enum
+from .Mssql import MsSqlEngine
+from .Pgsql import PgSqlEngine
+from .Mongo import MongoEngine

core/utils/te/utils/Encoder.py

@@ -0,0 +1,19 @@
+import json
+
+from sqlalchemy.orm import DeclarativeMeta
+
+
+class AlchemyEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj.__class__, DeclarativeMeta):
+            fields = {}
+            for field in [x for x in dir(obj) if not x.startswith('_') and x != 'metadata']:
+                data = obj.__getattribute__(field)
+                try:
+                    json.dumps(data)
+                    fields[field] = data
+                except TypeError:
+                    fields[field] = None
+            return fields
+
+        return json.JSONEncoder.default(self, obj)

core/utils/te/utils/Models.py

@@ -0,0 +1,160 @@
+from enum import Enum
+from typing import Optional, Any, List
+
+import pydantic
+from pydantic import BaseModel
+
+
+class FilterItem(BaseModel):
+    col: Optional[str]
+    cr: Optional[str]
+    id: Optional[str]
+    op: Optional[str]
+    type: Optional[str]
+    val: Optional[Any]
+
+    @staticmethod
+    def ValidItem(item: dict):
+        return item.get('col') and \
+            item.get('id') and \
+            item.get('op') and \
+            item.get('type')
+
+    @staticmethod
+    def ValidValue(item: dict):
+        return len(str(item.get('val', '')).strip()) > 0
+
+
+class Cr(str, Enum):
+    OR = 'or'
+    AND = 'and'
+
+
+class FilterScope(BaseModel):
+    id: str
+    items: Optional[List[FilterItem]]
+    op: Optional[Cr]
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        if kwargs.get('items'):
+            self.items = [
+                FilterItem(**item) for item in filter(lambda i: FilterItem.ValidItem(i), kwargs.get('items', []))]
+        self.op = Cr(kwargs.get('op'))
+        self.id = str(kwargs.get('id'))
+
+    @staticmethod
+    def ValidScope(item: dict):
+        return item.get('id') and item.get('items') and len(
+            item.get('items')) > 0 and item.get('op')
+
+    @staticmethod
+    def CleanItems(item):
+        clone = item.dict()
+        clone['items'] = list(filter(
+            lambda i: FilterItem.ValidValue(i), clone.get(
+                'items', [])))
+        return FilterScope(**clone)
+
+
+class Page(BaseModel):
+    c: int = 1
+    s: int = 5
+
+
+class SearchModel(BaseModel):
+    text: Optional[str]
+    cols: Optional[List[Any]]
+
+
+class SortModel(BaseModel):
+    field: Optional[str]
+    direction: Optional[str]
+
+
+class JoinModel(BaseModel):
+    model: BaseModel
+    show: List[str]
+    search: Optional[List[str]]
+
+
+class Config:
+    extra = pydantic.Extra.ignore
+    any_str_strip_whitespace = True  # strip whitespaces from strings
+    underscore_attrs_are_private = False
+    arbitrary_types_allowed = True
+
+
+class TableEngineInput(BaseModel):
+    f: Optional[List[FilterScope]]
+    extra_filter: list = []
+    o: Optional[SortModel]
+    p: Optional[Page] = Page()
+    s: Optional[SearchModel]
+    is_deleted: Optional[bool]
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        _input = {**{'p': {'c': 1, 's': 5},
+                     'f': [], 's': {}, 'o': {}}, **(kwargs or {})}
+        self.p = Page(**_input.get('p'))
+        self.o = SortModel(**_input.get('o'))
+        if _input.get('f'):
+            self.f = [FilterScope(**f) for f in
+                      filter(lambda fs: FilterScope.ValidScope(fs), _input.get('f'))]
+            self.f = list(map(lambda fs: FilterScope.CleanItems(fs), self.f))
+        if _input.get('s'):
+            self.s = SearchModel(**_input.get('s'))
+
+    def limit(self):
+        return self.p.s
+
+    def order(self):
+        return self.o
+
+    def filters(self) -> List[FilterScope]:
+        return self.f
+
+    def has_filter(self) -> bool:
+        return self.f is not None and \
+            len(self.f) > 0 and \
+            self.f[0].items is not None and \
+            len(self.f[0].items) > 0
+
+    def has_extra_filter(self) -> bool:
+        return self.extra_filter is not None and \
+            len(self.extra_filter) > 0
+
+    def skip(self):
+        return self.page() * self.p.s
+
+    def page(self):
+        return self.p.c - 1
+
+    def virtual_page(self):
+        return self.page() + 1
+
+    def get_limit(self):
+        limit = self.p.s if self.p else 20
+        return limit if limit < 100 else 100
+
+    def text_search_cols(self, cols: List[str]):
+        if self.s:
+            self.s.cols = cols
+
+    def has_text_search(self) -> bool:
+        return self.s is not None and self.s.text is not None and len(self.s.text.strip()) > 0
+
+    def add_filter(self, _filter):
+        self.extra_filter.append(_filter)
+
+    def get_search_cols(self) -> List[str]:
+        return self.s.cols if self.s else []
+
+    def get_text_search(self) -> str:
+        return self.s.text.strip() if self.s else None
+
+
+class WorkingMode(Enum):
+    Development = "development"
+    Production = "production"

core/utils/utils.py

@@ -0,0 +1,150 @@
+import re
+from datetime import datetime
+from functools import reduce
+from time import time
+from uuid import uuid4, UUID
+
+import pytz as pytz
+import shortuuid
+from fastapi.requests import Request
+
+
+def get_time() -> datetime:
+    """
+    Returns the current time.
+    Returns:
+    """
+    return datetime.fromtimestamp(time(), tz=pytz.timezone('Asia/Tehran'))
+
+
+def get_uuid() -> UUID:
+    """Returns an unique UUID (UUID4)"""
+    return uuid4()
+
+
+def is_numeric(_input) -> bool:
+    """Returns True if the input is a number."""
+    try:
+        val = int(_input)
+        return True
+    except ValueError:
+        return False
+
+
+def get_v_id(prefix: str, prefix_len=2, source_id: UUID = None) -> str:
+    main_id = source_id if source_id is not None else uuid4()
+    short_id = shortuuid.encode(main_id)
+    return str(prefix)[0:prefix_len] + '.' + short_id[:7]
+
+
+def get_qr_v_id(source_id: UUID = None) -> str:
+    main_id = source_id if source_id is not None else uuid4()
+    short_id = shortuuid.encode(main_id)
+    return short_id[:8]
+
+
+def is_valid_uuid(uuid_to_test, version=4):
+    """
+    Check if uuid_to_test is a valid UUID.
+
+     Parameters
+    ----------
+    uuid_to_test : str
+    version : {1, 2, 3, 4}
+
+     Returns
+    -------
+    `True` if uuid_to_test is a valid UUID, otherwise `False`.
+
+     Examples
+    --------
+    >>> is_valid_uuid('c9bf9e57-1685-4c89-bafb-ff5af830be8a')
+    True
+    >>> is_valid_uuid('c9bf9e58')
+    False
+    """
+
+    try:
+        uuid_obj = UUID(str(uuid_to_test), version=version)
+    except ValueError:
+        return False
+    return str(uuid_obj) == str(uuid_to_test)
+
+
+def get_base_url(request: Request) -> str:
+    """
+    Returns the base url of the request.
+
+    Parameters
+    ----------
+    request : Request
+
+    Returns
+    -------
+    str
+    """
+    return '{0}://{1}{2}/'.format(request.url.scheme,
+                                  request.url.hostname,
+                                  ':' + request.url.port.__str__() if request.url.port else '')
+
+
+def p2e_numbers(input: str) -> str:
+    """
+    convert persian numbers to english numbers.
+    """
+    _map = {
+        '۱': '1',
+        '۲': '2',
+        '۳': '3',
+        '۴': '4',
+        '۵': '5',
+        '۶': '6',
+        '۷': '7',
+        '۸': '8',
+        '۹': '9',
+        '۰': '0'
+    }
+    a = map(lambda ch: _map[ch] if ch in _map else ch, input)
+    return ''.join(list(a))
+
+
+def is_list_of_strings(lst):
+    return bool(lst) and isinstance(lst, list) and all(isinstance(elem, str) for elem in lst)
+
+
+def smart_cast(_input):
+    if is_numeric(_input):
+        return int(_input)
+    if is_valid_uuid(_input):
+        return UUID(str(_input))
+    return _input
+
+
+def is_mobile_number(mobile: str):
+    regex = r"^\+?[0-0][0-9]{10}$"
+    if mobile and not re.search(regex, mobile, re.I):
+        return False
+    return True
+
+
+def is_email(email: str):
+    regex = r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b'
+    if email and not re.search(regex, email, re.I):
+        return False
+    return True
+
+
+def deep_get(dictionary, keys, default=None):
+    return reduce(lambda d, key: d.get(key, default) if isinstance(d, dict) else default, keys.split("."), dictionary)
+
+
+def get_tag_strips(tag_name: str, content: str):
+    try:
+        r_string = fr'\[{tag_name}\](.|\n)*?\[\/{tag_name}\]'
+        matches = re.search(r_string, str(content), re.MULTILINE)
+        raw = matches.group(0)
+        raw = raw.replace(f'[{tag_name}]', '')
+        raw = raw.replace(f'[/{tag_name}]', '')
+        return raw
+    except BaseException:
+        return False