ms.shahzeidi/sakhteman
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(project):change mongo

Browse Source
This commit is contained in:
shahzeydyfah
2025-01-10 22:02:00 +03:30
parent a83d979a54
commit 0630f04690
5 changed files with 292 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/utils/te/__init__.py
+22 -6
View File
@@ -1,14 +1,16 @@
from typing import List
from starlette.responses import FileResponse
from .engines import MsSqlEngine, MongoEngine, PgSqlEngine, AbstractEngine
from .utils.Models import TableEngineInput
from .utils.Models import WorkingMode, JoinModel
from ..table_engine import TableEngineExport
from .utils.Models import TableEngineExport
class NewTableEngine:
def __init__(self, query_class, model, inputs, mode, engine: AbstractEngine, extra_filter,
default_sort_by, joins, search_cols, options=None, soft_deleted=False):
default_sort_by, joins, search_cols, options=None, soft_deleted=False, unselect={}, return_query=False):
self.model = model
self.query_class = query_class
self.mode = mode
@@ -18,12 +20,15 @@ class NewTableEngine:
self.input = TableEngineInput(**inputs if inputs else {})
else:
self.input = TableEngineInput(**inputs.dict() if inputs else {})
self.input.extra_filter = extra_filter
self.default_sort_by = default_sort_by
self.joins = joins
self.soft_deleted = soft_deleted
self.options = options
self.search_cols = search_cols
self.unselect = unselect
self.return_query = return_query
self.engine = engine(self)
@staticmethod
@@ -33,10 +38,10 @@ class NewTableEngine:
mode: WorkingMode = WorkingMode.Development,
extra_filter=None,
soft_deleted=False,
default_sort_by=None, joins=None, search_cols=[]):
default_sort_by=None, joins=None, search_cols=[] ,unselect={}, return_query=False):
return NewTableEngine(query_class, model, inputs, mode, engine=MongoEngine, extra_filter=extra_filter,
default_sort_by=default_sort_by, joins=joins, search_cols=search_cols,
soft_deleted=soft_deleted)
soft_deleted=soft_deleted,unselect=unselect, return_query=return_query)
@staticmethod
def mssql(query_class, model, inputs, mode: WorkingMode = WorkingMode.Development, extra_filter=None,
@@ -53,15 +58,19 @@ class NewTableEngine:
async def exe(self):
return await self.engine.exe()
async def to_dict(self) -> TableEngineExport:
async def to_dict(self):
if self.return_query:
query = await self.exe()
return query
rows, count = await self.exe()
self.input.extra_filter = None
del self.engine
self.set_export_file()
return TableEngineExport(
rows=rows,
page=self.input.virtual_page(),
total=count,
te=self.input
te=self.raw_input
)
def add_extra_filter(self, filters):
@@ -75,3 +84,10 @@ class NewTableEngine:
def set_custom_entities(self, entities=None):
self.engine.custom_entities = entities
return self
def set_export_file(self):
try:
self.raw_input['e']['d'] = self.input.e.d
return self
except Exception as e:
return False
core/utils/te/engines/AbstractEngine.py
+2
View File
@@ -15,6 +15,8 @@ class AbstractEngineClass(abc.ABC):
self.options = table_engine.options
self.search_cols = table_engine.search_cols
self.soft_deleted = table_engine.soft_deleted
self.unselect = table_engine.unselect
self.return_query = table_engine.return_query
@abc.abstractmethod
def exe(self):
core/utils/te/engines/Mongo.py
+102 -18
View File
@@ -1,9 +1,30 @@
import datetime
import uuid
from typing import Union, List
from uuid import UUID
# import dateutil.parser
from celery import shared_task
# from dateutil.relativedelta import relativedelta
from .AbstractEngine import AbstractEngineClass
from ..utils.Models import FilterItem, FilterScope
from ..utils.Models import FilterItem, FilterScope, OutPutType
# def zero_time_data(date, is_max=False):
# _date = dateutil.parser.isoparse(date)
# return datetime.datetime.combine(_date, datetime.time.min if not is_max else datetime.time.max)
# def date_year_month_morph(date, is_max=False, has_month=True):
# _date = date
# if len(date) > 7:
# _date = dateutil.parser.isoparse(date)
# if not has_month and len(date) < 5:
# _date = f'{date}-01'
# _date = datetime.datetime.strptime(_date, '%Y-%m')
# _date = _date + relativedelta(day=31)
# return _date if is_max else _date + relativedelta(day=1)
class MongoEngine(AbstractEngineClass):
@@ -13,15 +34,31 @@ class MongoEngine(AbstractEngineClass):
async def exe(self):
_f = self.filters = self.set_filters()
tmp = self.query_class.find(_f).sort(**self.set_sort())
print("_f ====>", _f)
if self.unselect:
tmp = self.query_class.find(_f, self.unselect)
else:
tmp = self.query_class.find(_f)
sort_by = self.set_sort()
if isinstance(sort_by, dict):
tmp = tmp.sort(**sort_by)
else:
tmp = tmp.sort(sort_by)
self.rows = tmp.limit(
self.input.get_limit()).skip(self.input.skip())
if self.input.e and self.input.e.ot:
self.input.e.d = self.generate_output(self.query_class, _f)
if self.return_query:
return {'query':_f, 'sort':self.set_sort(), 'limit':self.input.get_limit(), 'page':self.input.skip()}
return [self.model(**document) async for document in self.rows], await self.count()
def query_generator(self):
if not self.input.has_filter():
return []
filters = self.input.filters()
filters = [f for f in self.input.filters() if
(hasattr(f, 'items') and len(f.items)) > 0 or not hasattr(f, 'items')]
query = self.generate_scope_filter(filters, general=True)
return query
@@ -36,7 +73,7 @@ class MongoEngine(AbstractEngineClass):
self.input.get_search_cols() and len(self.input.get_search_cols())) else self.search_cols
if _search_cols:
for col in _search_cols:
items.append({col: {'$regex': self.input.get_text_search()}})
items.append({col: {'$regex': self.input.get_text_search(), '$options': 'i'}})
filters.append({"$or": items})
if self.input.extra_filter:
filters.extend(self.input.extra_filter)
@@ -48,13 +85,18 @@ class MongoEngine(AbstractEngineClass):
return self.filters
def set_sort(self):
if self.input.o.fields:
all_sort = []
for item in self.input.o.fields:
all_sort.append((item.field, 1 if item.direction == "asc" else -1))
return all_sort
if self.input.o.field:
return {'key_or_list': self.input.o.field,
'direction': 1 if self.input.o.direction == "asc" else -1}
elif self.default_sort_by:
return self.default_sort_by
else:
return {'key_or_list': 'created_at', 'direction': -1}
return {'key_or_list': '$natural', 'direction': -1}
@staticmethod
def filter_item_generator(item: FilterItem) -> dict:
@@ -64,13 +106,14 @@ class MongoEngine(AbstractEngineClass):
"$elemMatch": {
'field_id': UUID(item.col),
'value': {
**MongoEngine.get_expression_filter(item.op, MongoEngine.cast_value(item.val, item.type), )}
**MongoEngine.get_expression_filter(item.op, MongoEngine.cast_value(item.val, item.type),
field_type=item.type)}
}
}
}
return {item.col: {
**MongoEngine.get_expression_filter(col=item.col, op=item.op,
val=MongoEngine.cast_value(item.val, item.type), )}}
val=MongoEngine.cast_value(item.val, item.type), field_type=item.type)}}
@staticmethod
def is_valid_uuid(uuid_to_test, version=4):
@@ -80,9 +123,12 @@ class MongoEngine(AbstractEngineClass):
return False
return str(uuid_obj) == str(uuid_to_test)
@staticmethod
def cast_value(val: Union[str, List], type: str) -> any:
if val == [] or type == 'BOOL':
return val
if not val:
return None
if type in ['NUMBER', 'STACK', 'FLOAT']:
if isinstance(val, List):
return [float(x) for x in val]
@@ -97,20 +143,37 @@ class MongoEngine(AbstractEngineClass):
return UUID(str(val))
return val
@staticmethod
def get_expression_filter(col=None, op: str = 'eq', val: str = '') -> dict:
def get_expression_filter(col=None, op: str = 'eq', val: str = '', field_type: str = None) -> dict:
_op = MongoEngine.get_operator(op)
# Date Type Operator
if field_type in ['DATE', 'DATE_TIME', 'DATETIME', 'DATE_MONTH', 'DATE_YEAR']:
if op == "eq":
return {'$gte': val[0], '$lte': val[1]}
if op == "n_eq":
return {'$lte': val[0], '$gte': val[1]}
if op == "lt":
return {'$lt': val[1]}
if op == "gt":
return {'$gt': val[0]}
if op == "lte":
return {'$lt': val[1]}
if op == "gte":
return {'$gt': val[0]}
if _op == "sw":
return {'$regex': f'^{val}'}
return {'$regex': f'^{val}', '$options': 'i'}
elif _op == "ew":
return {'$regex': f'{val}$'}
return {'$regex': f'{val}$', '$options': 'i'}
elif _op == "n_ew":
return {'$regex': f'(?<!{val})$'}
return {'$regex': f'(?<!{val})$', '$options': 'i'}
elif _op == "n_sw":
return {'$regex': f'^(?!{val})'}
return {'$regex': f'^(?!{val})', '$options': 'i'}
elif _op == "n_like":
return {'$not': {'$regex': val}}
return {'$not': {'$regex': val, '$options': 'i'}}
elif _op == "like":
return {'$regex': val, '$options': 'i'}
elif _op == "bet":
if isinstance(val, str):
return {'$gte': val.split('-')[0], '$lte': val.split('-')[1]}
@@ -124,9 +187,9 @@ class MongoEngine(AbstractEngineClass):
elif _op == "n_exi":
return {'$exists': False}
elif _op == "in":
return {'$exists': True}
return {'$in': val}
elif _op == "not_in":
return {'$exists': False}
return {'$nin': val}
return {_op: val}
@staticmethod
@@ -138,7 +201,7 @@ class MongoEngine(AbstractEngineClass):
"ge": "$ge",
"lt": "$lt",
"le": "$le",
"like": "$regex",
"like": "like",
"n_like": "n_like",
"sw": "sw",
"ew": "ew",
@@ -185,3 +248,24 @@ class MongoEngine(AbstractEngineClass):
def set_joins(self):
pass
def generate_output(self, query, _f):
print(_f.__dict__)
return ''
if self.input.e.ot == OutPutType.EXEL:
file_name = str(uuid.uuid4().hex) + '.xlsx'
return _f.dict()
MongoEngine.output_generator(str(query).replace('false', '0'), params,
str(self.query_class.bind.url), file_name=file_name, cols=self.input.e.c)
return file_name
elif self.input.e.ot == OutPutType.JSON:
pass
elif self.input.e.ot == OutPutType.PDF:
pass
else:
raise Exception("Output type not supported.")
@staticmethod
@shared_task(name='output_generator_task_mongo')
def output_generator(query, params, url, file_name, cols):
pass
core/utils/te/engines/Mssql.py
+112 -11
View File
@@ -1,8 +1,18 @@
from sqlalchemy import or_, and_, desc, asc
import os
import string
import tempfile
import uuid
# import xlsxwriter
from celery import shared_task
from sqlalchemy import or_, and_, desc, asc, create_engine, LABEL_STYLE_TABLENAME_PLUS_COL
from sqlalchemy.orm import sessionmaker
from sqlalchemy.sql.elements import BooleanClauseList, BinaryExpression, UnaryExpression
from sqlalchemy.sql.selectable import TableValuedAlias
from .AbstractEngine import AbstractEngineClass
from ..utils.Models import FilterItem, FilterScope
# from .Mongo import zero_time_data
from ..utils.Models import FilterItem, FilterScope, OutPutType
class MsSqlEngine(AbstractEngineClass):
@@ -16,7 +26,8 @@ class MsSqlEngine(AbstractEngineClass):
async def exe(self):
_f = self.set_filters()
if not self.custom_entities:
print(_f)
if self.custom_entities is not None:
query = self.query_class.query(self.model)
else:
query = self.query_class.query(*self.custom_entities)
@@ -29,9 +40,11 @@ class MsSqlEngine(AbstractEngineClass):
if self.options:
query = query.options(*self.options)
if self.input.e and self.input.e.ot:
self.input.e.d = self.generate_output(query, _f)
tmp = query.filter(_f).order_by(self.set_sort())
self.rows = tmp.limit(self.input.get_limit()).offset(self.input.skip()).all()
return self.rows, tmp.count(), tmp
return self.rows, tmp.count()
def query_generator(self):
if not self.input.has_filter():
@@ -42,8 +55,9 @@ class MsSqlEngine(AbstractEngineClass):
def set_sort(self):
if self.input.o.field:
self.sort = asc(getattr(self.model, self.input.o.field)) if self.input.o.direction == 'asc' else desc(
getattr(self.model, self.input.o.field))
sm = self.model.c if isinstance(self.model, TableValuedAlias) else self.model
self.sort = asc(getattr(sm, self.input.o.field)) if self.input.o.direction == 'asc' else desc(
getattr(sm, self.input.o.field))
return self.sort
elif isinstance(self.default_sort_by, UnaryExpression):
return self.default_sort_by
@@ -65,9 +79,13 @@ class MsSqlEngine(AbstractEngineClass):
if has_join:
pass
else:
sm = self.model.c if isinstance(self.model, TableValuedAlias) else self.model
items.append(
getattr(self.model, col).ilike("%{search}%".format(search=self.input.get_text_search())))
getattr(sm, col).ilike(
"%{search}%".format(search=self.input.get_text_search())))
filters.append(or_(*items))
if self.input.extra_filter:
for f in self.input.extra_filter:
filters.append(f)
@@ -88,7 +106,7 @@ class MsSqlEngine(AbstractEngineClass):
"n_eq": "__ne__",
"gt": "__gt__",
"gte": "__ge__",
"lte": "__lte__",
"lte": "__le__",
"lt": "__lt__",
"like": "ilike",
"n_like": "notlike",
@@ -115,10 +133,22 @@ class MsSqlEngine(AbstractEngineClass):
def filter_item_generator(self, item: FilterItem):
has_join = len(item.col.split('.')) > 1
if has_join:
pass
# pass
return self.join_filter(item)
else:
return getattr(getattr(self.model, item.col), self.get_operator(item.op))(item.val)
sm = self.model.c if isinstance(self.model, TableValuedAlias) else self.model
# if item.type in ['DATE', 'DATE_TIME']:
# if item.op == 'eq':
# return getattr(sm, item.col).between(zero_time_data(item.val),
# zero_time_data(item.val, is_max=True))
# if item.op in ['gte', 'gt']:
# item.val = zero_time_data(item.val)
# print(item.val,'GT')
# if item.op in ['lt', 'lte']:
# item.val = zero_time_data(item.val, is_max=True)
# print(item.val,'LT')
return getattr(getattr(sm, item.col), self.get_operator(item.op))(item.val)
def generate_scope_filter(self, filters: list, general=False):
if len(filters) == 1:
@@ -144,3 +174,74 @@ class MsSqlEngine(AbstractEngineClass):
filters = [_op(_a, _b)] + filters
return self.generate_scope_filter(filters, general)
def join_filter(self, item: FilterItem):
inner_joins = item.col.split('.')[-2:]
join = None
for self_join in self.joins:
if inner_joins[0] == self_join.__name__ or self.normalize_model_name(inner_joins[0]) == self_join.__name__:
join = self_join
return getattr(getattr(join, inner_joins[1]), self.get_operator(item.op))(item.val) if join else None
@staticmethod
def normalize_model_name(obj):
if isinstance(obj, str):
return obj.replace('_', '')
else:
result = {}
model_name = str([i for i in obj.keys()][0]).rsplit('_', 1)
model_name.pop(-1)
model_name = ''.join(model_name)
for index in obj.keys():
item = index
for i in range(5):
item = item.replace(f'_{i}_', '_')
if item[-1] == '_':
item = item[:-1]
item = item.replace(model_name + '_', '')
result[item] = obj[index]
return result
def generate_output(self, query, _f):
if self.input.e.ot == OutPutType.EXEL:
tmp = query.filter(_f).order_by(self.set_sort()).set_label_style(LABEL_STYLE_TABLENAME_PLUS_COL)
query = tmp.statement.compile()
params = query.params
file_name = str(uuid.uuid4().hex) + '.xlsx'
MsSqlEngine.output_generator(str(query).replace('false', '0'), params,
str(self.query_class.bind.url), file_name=file_name, cols=self.input.e.c)
return file_name
elif self.input.e.ot == OutPutType.JSON:
pass
elif self.input.e.ot == OutPutType.PDF:
pass
else:
raise Exception("Output type not supported.")
@staticmethod
@shared_task(name='output_generator_task')
def output_generator(query, params, url, file_name, cols):
# engine = create_engine(url=url)
# session = sessionmaker(autocommit=False, autoflush=False, bind=engine)
# db = session()
# datas = db.execute(query, params)
# file_dir = os.path.join(tempfile.gettempdir(), file_name)
# head_cols = list(str(string.ascii_uppercase))
# headers = cols.keys()
# # workbook = xlsxwriter.Workbook(file_dir)
# worksheet = workbook.add_worksheet()
# for header in headers:
# worksheet.write(f'{head_cols.pop(0)}1', cols[header])
# row = 1
# for item in datas:
# row += 1
# head_cols = list(str(string.ascii_uppercase))
# data = MsSqlEngine.normalize_model_name(item)
# for col in headers:
# if col[-1] == '_':
# col = col[:-1]
# col = col.replace('.', '_')
# worksheet.write(f'{head_cols.pop(0)}{row}', data[col] if col in data else '-')
# workbook.close()
...
return True
core/utils/te/utils/Models.py
+54 -4
View File
@@ -1,8 +1,10 @@
from enum import Enum
from typing import Optional, Any, List
from typing import Any
from typing import Generic, TypeVar, List, Optional
import pydantic
from pydantic import BaseModel
from pydantic.generics import GenericModel
class FilterItem(BaseModel):
@@ -12,6 +14,7 @@ class FilterItem(BaseModel):
op: Optional[str]
type: Optional[str]
val: Optional[Any]
hidden: Optional[bool]
@staticmethod
def ValidItem(item: dict):
@@ -30,6 +33,12 @@ class Cr(str, Enum):
AND = 'and'
class OutPutType(str, Enum):
JSON = 'json'
EXEL = 'exel'
PDF = 'pdf'
class FilterScope(BaseModel):
id: str
items: Optional[List[FilterItem]]
@@ -60,6 +69,7 @@ class FilterScope(BaseModel):
class Page(BaseModel):
c: int = 1
s: int = 5
force: Optional[bool] = False
class SearchModel(BaseModel):
@@ -67,11 +77,23 @@ class SearchModel(BaseModel):
cols: Optional[List[Any]]
class ExportModel(BaseModel):
ot: Optional[OutPutType]
c: Optional[dict]
d: Optional[str]
class SortModel(BaseModel):
field: Optional[str]
direction: Optional[str]
class MultiSortModel(BaseModel):
field: Optional[str]
direction: Optional[str]
fields: Optional[List[SortModel]]
class JoinModel(BaseModel):
model: BaseModel
show: List[str]
@@ -88,17 +110,19 @@ class Config:
class TableEngineInput(BaseModel):
f: Optional[List[FilterScope]]
extra_filter: list = []
o: Optional[SortModel]
o: Optional[MultiSortModel]
p: Optional[Page] = Page()
s: Optional[SearchModel]
e: Optional[ExportModel]
is_deleted: Optional[bool]
def __init__(self, **kwargs):
super().__init__(**kwargs)
_input = {**{'p': {'c': 1, 's': 5},
'f': [], 's': {}, 'o': {}}, **(kwargs or {})}
'f': [], 's': {}, 'o': {}, 'e': {}}, **(kwargs or {})}
self.p = Page(**_input.get('p'))
self.o = SortModel(**_input.get('o'))
self.o = MultiSortModel(**_input.get('o'))
self.e = ExportModel(**_input.get('e'))
if _input.get('f'):
self.f = [FilterScope(**f) for f in
filter(lambda fs: FilterScope.ValidScope(fs), _input.get('f'))]
@@ -135,7 +159,10 @@ class TableEngineInput(BaseModel):
return self.page() + 1
def get_limit(self):
# HIT : LIMIT 0 : in Mongo Return all records
limit = self.p.s if self.p else 20
if self.p.force:
return 1000000
return limit if limit < 100 else 100
def text_search_cols(self, cols: List[str]):
@@ -158,3 +185,26 @@ class TableEngineInput(BaseModel):
class WorkingMode(Enum):
Development = "development"
Production = "production"
T = TypeVar('T')
class TableEngineInputType(GenericModel, Generic[T]):
f: Optional[List]
o: Optional[dict] = {}
p: Optional[dict] = {}
s: Optional[dict] = {}
e: Optional[dict] = {}
class TableEngineExport(GenericModel, Generic[T]):
rows: Optional[List[T]]
total: Optional[int]
page: Optional[int]
te: Optional[TableEngineInputType]
class TableEngineRequestType(GenericModel, Generic[T]):
te: Optional[TableEngineInputType]
extra: Optional[dict]