Initial commit

This commit is contained in:
shahzeydyfah
2024-02-15 00:17:51 +03:30
commit 4b0396466d
111 changed files with 6296 additions and 0 deletions
+77
View File
@@ -0,0 +1,77 @@
from typing import List
from .engines import MsSqlEngine, MongoEngine, PgSqlEngine, AbstractEngine
from .utils.Models import TableEngineInput
from .utils.Models import WorkingMode, JoinModel
from ..table_engine import TableEngineExport
class NewTableEngine:
def __init__(self, query_class, model, inputs, mode, engine: AbstractEngine, extra_filter,
default_sort_by, joins, search_cols, options=None, soft_deleted=False):
self.model = model
self.query_class = query_class
self.mode = mode
self.joins = False
self.raw_input = inputs
if isinstance(inputs, dict):
self.input = TableEngineInput(**inputs if inputs else {})
else:
self.input = TableEngineInput(**inputs.dict() if inputs else {})
self.input.extra_filter = extra_filter
self.default_sort_by = default_sort_by
self.joins = joins
self.soft_deleted = soft_deleted
self.options = options
self.search_cols = search_cols
self.engine = engine(self)
@staticmethod
def mongo(query_class,
model,
inputs,
mode: WorkingMode = WorkingMode.Development,
extra_filter=None,
soft_deleted=False,
default_sort_by=None, joins=None, search_cols=[]):
return NewTableEngine(query_class, model, inputs, mode, engine=MongoEngine, extra_filter=extra_filter,
default_sort_by=default_sort_by, joins=joins, search_cols=search_cols,
soft_deleted=soft_deleted)
@staticmethod
def mssql(query_class, model, inputs, mode: WorkingMode = WorkingMode.Development, extra_filter=None,
default_sort_by=None, joins=None, search_cols=[], options=None):
return NewTableEngine(query_class, model, inputs, mode, engine=MsSqlEngine, extra_filter=extra_filter,
default_sort_by=default_sort_by, joins=joins, search_cols=search_cols, options=options)
@staticmethod
def pgsql(query_class, model, inputs, mode: WorkingMode = WorkingMode.Development, extra_filter=None,
default_sort_by=None, joins=None, search_cols=None, options=None):
return NewTableEngine(query_class, model, inputs, mode, engine=PgSqlEngine, extra_filter=extra_filter,
default_sort_by=default_sort_by, joins=joins, search_cols=search_cols, options=options)
async def exe(self):
return await self.engine.exe()
async def to_dict(self) -> TableEngineExport:
rows, count = await self.exe()
self.input.extra_filter = None
del self.engine
return TableEngineExport(
rows=rows,
page=self.input.virtual_page(),
total=count,
te=self.input
)
def add_extra_filter(self, filters):
self.input = filters
return self
def joins(self, joins: List[JoinModel]):
self.joins = joins
return self
def set_custom_entities(self, entities=None):
self.engine.custom_entities = entities
return self
+51
View File
@@ -0,0 +1,51 @@
import abc
__all__ = "AbstractEngine"
class AbstractEngineClass(abc.ABC):
def __init__(self, table_engine):
self.model = table_engine.model
self.query_class = table_engine.query_class
self.input = table_engine.input
self.filter = table_engine.input.f
self.default_sort_by = table_engine.default_sort_by
self.custom_entities = []
self.joins = table_engine.joins
self.options = table_engine.options
self.search_cols = table_engine.search_cols
self.soft_deleted = table_engine.soft_deleted
@abc.abstractmethod
def exe(self):
pass
@abc.abstractmethod
def set_filters(self):
pass
@abc.abstractmethod
def set_sort(self):
pass
@staticmethod
@abc.abstractmethod
def get_expression_filter(col=None, op: str = 'eq', val: str = '') -> dict:
pass
@staticmethod
@abc.abstractmethod
def get_operator(op: str):
pass
def count(self):
pass
@staticmethod
@abc.abstractmethod
def generate_scope_filter(filters: list, general=False):
pass
@abc.abstractmethod
def set_joins(self):
pass
+187
View File
@@ -0,0 +1,187 @@
import datetime
from typing import Union, List
from uuid import UUID
from .AbstractEngine import AbstractEngineClass
from ..utils.Models import FilterItem, FilterScope
class MongoEngine(AbstractEngineClass):
rows = []
stack = []
filters = {}
async def exe(self):
_f = self.filters = self.set_filters()
tmp = self.query_class.find(_f).sort(**self.set_sort())
self.rows = tmp.limit(
self.input.get_limit()).skip(self.input.skip())
return [self.model(**document) async for document in self.rows], await self.count()
def query_generator(self):
if not self.input.has_filter():
return []
filters = self.input.filters()
query = self.generate_scope_filter(filters, general=True)
return query
def set_filters(self):
filters = []
query = self.query_generator()
if query:
filters.append(query)
if self.input.has_text_search():
items = []
_search_cols = self.input.get_search_cols() if (
self.input.get_search_cols() and len(self.input.get_search_cols())) else self.search_cols
if _search_cols:
for col in _search_cols:
items.append({col: {'$regex': self.input.get_text_search()}})
filters.append({"$or": items})
if self.input.extra_filter:
filters.extend(self.input.extra_filter)
if self.soft_deleted:
filters.append({'deleted_at': {'$exists': False}})
if not filters:
return {}
self.filters = {'$and': filters}
return self.filters
def set_sort(self):
if self.input.o.field:
return {'key_or_list': self.input.o.field,
'direction': 1 if self.input.o.direction == "asc" else -1}
elif self.default_sort_by:
return self.default_sort_by
else:
return {'key_or_list': 'created_at', 'direction': -1}
@staticmethod
def filter_item_generator(item: FilterItem) -> dict:
if MongoEngine.is_valid_uuid(str(item.col)):
return {
'data': {
"$elemMatch": {
'field_id': UUID(item.col),
'value': {
**MongoEngine.get_expression_filter(item.op, MongoEngine.cast_value(item.val, item.type), )}
}
}
}
return {item.col: {
**MongoEngine.get_expression_filter(col=item.col, op=item.op,
val=MongoEngine.cast_value(item.val, item.type), )}}
@staticmethod
def is_valid_uuid(uuid_to_test, version=4):
try:
uuid_obj = UUID(str(uuid_to_test), version=version)
except ValueError:
return False
return str(uuid_obj) == str(uuid_to_test)
@staticmethod
def cast_value(val: Union[str, List], type: str) -> any:
if type in ['NUMBER', 'STACK', 'FLOAT']:
if isinstance(val, List):
return [float(x) for x in val]
return float(val)
if type in ['DATE', 'DATETIME']:
if isinstance(val, List):
return [datetime.datetime.fromisoformat(str(x)) for x in val]
return datetime.datetime.fromisoformat(val)
if type == 'UUID':
if isinstance(val, List):
return [UUID(str(x)) for x in val]
return UUID(str(val))
return val
@staticmethod
def get_expression_filter(col=None, op: str = 'eq', val: str = '') -> dict:
_op = MongoEngine.get_operator(op)
if _op == "sw":
return {'$regex': f'^{val}'}
elif _op == "ew":
return {'$regex': f'{val}$'}
elif _op == "n_ew":
return {'$regex': f'(?<!{val})$'}
elif _op == "n_sw":
return {'$regex': f'^(?!{val})'}
elif _op == "n_like":
return {'$not': {'$regex': val}}
elif _op == "bet":
if isinstance(val, str):
return {'$gte': val.split('-')[0], '$lte': val.split('-')[1]}
return {'$gte': val[0], '$lte': val[1]}
elif _op == "n_bet":
if isinstance(val, str):
return {'$gte': val.split('-')[0], '$lte': val.split('-')[1]}
return {'$not': {'$gte': val[0], '$lte': val[1]}}
elif _op == "exi":
return {'$exists': True}
elif _op == "n_exi":
return {'$exists': False}
elif _op == "in":
return {'$exists': True}
elif _op == "not_in":
return {'$exists': False}
return {_op: val}
@staticmethod
def get_operator(op: str) -> str:
ops = {
"eq": "$eq",
"n_eq": "$ne",
"gt": "$gt",
"ge": "$ge",
"lt": "$lt",
"le": "$le",
"like": "$regex",
"n_like": "n_like",
"sw": "sw",
"ew": "ew",
'n_ew': 'n_ew',
'n_sw': 'n_sw',
'exi': 'exi',
"n_exi": "n_exi",
'lte': '$lte',
'gte': '$gte',
'in': 'in',
'n_in': 'not_in',
'bet': 'bet',
'n_bet': 'n_bet',
}
return ops[op] if op in ops else '$eq'
async def count(self):
return await self.query_class.count_documents(self.filters)
@staticmethod
def generate_scope_filter(filters: list, general=False):
if len(filters) == 1:
_o = filters[0]
if isinstance(_o, FilterItem):
return MongoEngine.filter_item_generator(filters[0])
elif isinstance(_o, FilterScope):
return MongoEngine.generate_scope_filter(_o.items)
else:
return _o
else:
_a = filters.pop(0)
_a = (MongoEngine.filter_item_generator(_a) if isinstance(_a,
FilterItem) else _a) if not general else (
MongoEngine.generate_scope_filter(_a.items) if isinstance(_a, FilterScope) else _a)
_b = filters.pop(0)
op = '$' + (_b.cr if not general else _b.op)
_b = (MongoEngine.filter_item_generator(_b) if isinstance(_b,
FilterItem) else _b) if not general else (
MongoEngine.generate_scope_filter(_b.items) if isinstance(_b, FilterScope) else _b)
filters = [{op: [
_a, _b
]}] + filters
return MongoEngine.generate_scope_filter(filters, general)
def set_joins(self):
pass
+146
View File
@@ -0,0 +1,146 @@
from sqlalchemy import or_, and_, desc, asc
from sqlalchemy.sql.elements import BooleanClauseList, BinaryExpression, UnaryExpression
from .AbstractEngine import AbstractEngineClass
from ..utils.Models import FilterItem, FilterScope
class MsSqlEngine(AbstractEngineClass):
def set_joins(self):
pass
filters = {}
rows = []
sort = None
async def exe(self):
_f = self.set_filters()
if not self.custom_entities:
query = self.query_class.query(self.model)
else:
query = self.query_class.query(*self.custom_entities)
if self.joins:
for j in self.joins:
if isinstance(j, list):
query = query.join(*j)
else:
query = query.join(j)
if self.options:
query = query.options(*self.options)
tmp = query.filter(_f).order_by(self.set_sort())
self.rows = tmp.limit(self.input.get_limit()).offset(self.input.skip()).all()
return self.rows, tmp.count(), tmp
def query_generator(self):
if not self.input.has_filter():
return None
filters = self.input.filters()
query = self.generate_scope_filter(filters, general=True)
return query
def set_sort(self):
if self.input.o.field:
self.sort = asc(getattr(self.model, self.input.o.field)) if self.input.o.direction == 'asc' else desc(
getattr(self.model, self.input.o.field))
return self.sort
elif isinstance(self.default_sort_by, UnaryExpression):
return self.default_sort_by
else:
return True
def set_filters(self):
filters = []
query = self.query_generator()
if isinstance(query, BooleanClauseList) or isinstance(query, BinaryExpression):
filters.append(query)
if self.input.has_text_search():
items = []
_search_cols = self.input.get_search_cols() if (
self.input.get_search_cols() and len(self.input.get_search_cols())) else self.search_cols
if _search_cols:
for col in _search_cols:
has_join = len(col.split('.')) > 1
if has_join:
pass
else:
items.append(
getattr(self.model, col).ilike("%{search}%".format(search=self.input.get_text_search())))
filters.append(or_(*items))
if self.input.extra_filter:
for f in self.input.extra_filter:
filters.append(f)
if not filters:
return True
self.filters = and_(*filters)
return self.filters
@staticmethod
def get_expression_filter(col=None, op: str = '', val: str = '') -> dict:
_op = MsSqlEngine.get_operator(op)
return getattr(col, op)(val)
@staticmethod
def get_operator(op: str):
ops = {
"eq": "__eq__",
"n_eq": "__ne__",
"gt": "__gt__",
"gte": "__ge__",
"lte": "__lte__",
"lt": "__lt__",
"like": "ilike",
"n_like": "notlike",
"in": "in_",
"n_in": "not_in",
"bet": "between",
"n_bet": "not_between",
"sw": "startswith",
"ew": "endswith",
'n_ew': 'not_endswith',
'n_sw': 'not_startswith',
'exi': 'exi',
"n_exi": "n_exi",
}
return ops[op] if op in ops else '__eq__'
@staticmethod
def get_and_or(op):
return or_ if op == 'or' else and_
def count(self):
pass
def filter_item_generator(self, item: FilterItem):
has_join = len(item.col.split('.')) > 1
if has_join:
pass
else:
return getattr(getattr(self.model, item.col), self.get_operator(item.op))(item.val)
def generate_scope_filter(self, filters: list, general=False):
if len(filters) == 1:
_o = filters[0]
if isinstance(_o, FilterItem):
return self.filter_item_generator(filters[0])
elif isinstance(_o, FilterScope):
return self.generate_scope_filter(_o.items)
else:
return _o
else:
_a = filters.pop(0)
_a = (self.filter_item_generator(_a) if isinstance(_a,
FilterItem) else _a) if not general else (
self.generate_scope_filter(_a.items) if isinstance(_a, FilterScope) else _a)
_b = filters.pop(0)
_op = MsSqlEngine.get_and_or(_b.cr if not general else _b.op)
_b = (self.filter_item_generator(_b) if isinstance(_b,
FilterItem) else _b) if not general else (
self.generate_scope_filter(_b.items) if isinstance(_b, FilterScope) else _b)
filters = [_op(_a, _b)] + filters
return self.generate_scope_filter(filters, general)
+13
View File
@@ -0,0 +1,13 @@
from .AbstractEngine import AbstractEngineClass
class PgSqlEngine(AbstractEngineClass):
def exe(self):
pass
def get_filters(self):
pass
def pagination(self):
pass
+4
View File
@@ -0,0 +1,4 @@
from enum import Enum
from .Mssql import MsSqlEngine
from .Pgsql import PgSqlEngine
from .Mongo import MongoEngine
+19
View File
@@ -0,0 +1,19 @@
import json
from sqlalchemy.orm import DeclarativeMeta
class AlchemyEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj.__class__, DeclarativeMeta):
fields = {}
for field in [x for x in dir(obj) if not x.startswith('_') and x != 'metadata']:
data = obj.__getattribute__(field)
try:
json.dumps(data)
fields[field] = data
except TypeError:
fields[field] = None
return fields
return json.JSONEncoder.default(self, obj)
+160
View File
@@ -0,0 +1,160 @@
from enum import Enum
from typing import Optional, Any, List
import pydantic
from pydantic import BaseModel
class FilterItem(BaseModel):
col: Optional[str]
cr: Optional[str]
id: Optional[str]
op: Optional[str]
type: Optional[str]
val: Optional[Any]
@staticmethod
def ValidItem(item: dict):
return item.get('col') and \
item.get('id') and \
item.get('op') and \
item.get('type')
@staticmethod
def ValidValue(item: dict):
return len(str(item.get('val', '')).strip()) > 0
class Cr(str, Enum):
OR = 'or'
AND = 'and'
class FilterScope(BaseModel):
id: str
items: Optional[List[FilterItem]]
op: Optional[Cr]
def __init__(self, **kwargs):
super().__init__(**kwargs)
if kwargs.get('items'):
self.items = [
FilterItem(**item) for item in filter(lambda i: FilterItem.ValidItem(i), kwargs.get('items', []))]
self.op = Cr(kwargs.get('op'))
self.id = str(kwargs.get('id'))
@staticmethod
def ValidScope(item: dict):
return item.get('id') and item.get('items') and len(
item.get('items')) > 0 and item.get('op')
@staticmethod
def CleanItems(item):
clone = item.dict()
clone['items'] = list(filter(
lambda i: FilterItem.ValidValue(i), clone.get(
'items', [])))
return FilterScope(**clone)
class Page(BaseModel):
c: int = 1
s: int = 5
class SearchModel(BaseModel):
text: Optional[str]
cols: Optional[List[Any]]
class SortModel(BaseModel):
field: Optional[str]
direction: Optional[str]
class JoinModel(BaseModel):
model: BaseModel
show: List[str]
search: Optional[List[str]]
class Config:
extra = pydantic.Extra.ignore
any_str_strip_whitespace = True # strip whitespaces from strings
underscore_attrs_are_private = False
arbitrary_types_allowed = True
class TableEngineInput(BaseModel):
f: Optional[List[FilterScope]]
extra_filter: list = []
o: Optional[SortModel]
p: Optional[Page] = Page()
s: Optional[SearchModel]
is_deleted: Optional[bool]
def __init__(self, **kwargs):
super().__init__(**kwargs)
_input = {**{'p': {'c': 1, 's': 5},
'f': [], 's': {}, 'o': {}}, **(kwargs or {})}
self.p = Page(**_input.get('p'))
self.o = SortModel(**_input.get('o'))
if _input.get('f'):
self.f = [FilterScope(**f) for f in
filter(lambda fs: FilterScope.ValidScope(fs), _input.get('f'))]
self.f = list(map(lambda fs: FilterScope.CleanItems(fs), self.f))
if _input.get('s'):
self.s = SearchModel(**_input.get('s'))
def limit(self):
return self.p.s
def order(self):
return self.o
def filters(self) -> List[FilterScope]:
return self.f
def has_filter(self) -> bool:
return self.f is not None and \
len(self.f) > 0 and \
self.f[0].items is not None and \
len(self.f[0].items) > 0
def has_extra_filter(self) -> bool:
return self.extra_filter is not None and \
len(self.extra_filter) > 0
def skip(self):
return self.page() * self.p.s
def page(self):
return self.p.c - 1
def virtual_page(self):
return self.page() + 1
def get_limit(self):
limit = self.p.s if self.p else 20
return limit if limit < 100 else 100
def text_search_cols(self, cols: List[str]):
if self.s:
self.s.cols = cols
def has_text_search(self) -> bool:
return self.s is not None and self.s.text is not None and len(self.s.text.strip()) > 0
def add_filter(self, _filter):
self.extra_filter.append(_filter)
def get_search_cols(self) -> List[str]:
return self.s.cols if self.s else []
def get_text_search(self) -> str:
return self.s.text.strip() if self.s else None
class WorkingMode(Enum):
Development = "development"
Production = "production"