From 0630f046909c9c8a78cc30d0563831ab302aea2a Mon Sep 17 00:00:00 2001 From: shahzeydyfah Date: Fri, 10 Jan 2025 22:02:00 +0330 Subject: [PATCH] feat(project):change mongo --- core/utils/te/__init__.py | 28 ++++-- core/utils/te/engines/AbstractEngine.py | 2 + core/utils/te/engines/Mongo.py | 120 +++++++++++++++++++---- core/utils/te/engines/Mssql.py | 123 +++++++++++++++++++++--- core/utils/te/utils/Models.py | 58 ++++++++++- 5 files changed, 292 insertions(+), 39 deletions(-) diff --git a/core/utils/te/__init__.py b/core/utils/te/__init__.py index b5c0928..c1e81c4 100644 --- a/core/utils/te/__init__.py +++ b/core/utils/te/__init__.py @@ -1,14 +1,16 @@ from typing import List +from starlette.responses import FileResponse + from .engines import MsSqlEngine, MongoEngine, PgSqlEngine, AbstractEngine from .utils.Models import TableEngineInput from .utils.Models import WorkingMode, JoinModel -from ..table_engine import TableEngineExport +from .utils.Models import TableEngineExport class NewTableEngine: def __init__(self, query_class, model, inputs, mode, engine: AbstractEngine, extra_filter, - default_sort_by, joins, search_cols, options=None, soft_deleted=False): + default_sort_by, joins, search_cols, options=None, soft_deleted=False, unselect={}, return_query=False): self.model = model self.query_class = query_class self.mode = mode @@ -18,12 +20,15 @@ class NewTableEngine: self.input = TableEngineInput(**inputs if inputs else {}) else: self.input = TableEngineInput(**inputs.dict() if inputs else {}) + self.input.extra_filter = extra_filter self.default_sort_by = default_sort_by self.joins = joins self.soft_deleted = soft_deleted self.options = options self.search_cols = search_cols + self.unselect = unselect + self.return_query = return_query self.engine = engine(self) @staticmethod @@ -33,10 +38,10 @@ class NewTableEngine: mode: WorkingMode = WorkingMode.Development, extra_filter=None, soft_deleted=False, - default_sort_by=None, joins=None, search_cols=[]): + default_sort_by=None, joins=None, search_cols=[] ,unselect={}, return_query=False): return NewTableEngine(query_class, model, inputs, mode, engine=MongoEngine, extra_filter=extra_filter, default_sort_by=default_sort_by, joins=joins, search_cols=search_cols, - soft_deleted=soft_deleted) + soft_deleted=soft_deleted,unselect=unselect, return_query=return_query) @staticmethod def mssql(query_class, model, inputs, mode: WorkingMode = WorkingMode.Development, extra_filter=None, @@ -53,15 +58,19 @@ class NewTableEngine: async def exe(self): return await self.engine.exe() - async def to_dict(self) -> TableEngineExport: + async def to_dict(self): + if self.return_query: + query = await self.exe() + return query rows, count = await self.exe() self.input.extra_filter = None del self.engine + self.set_export_file() return TableEngineExport( rows=rows, page=self.input.virtual_page(), total=count, - te=self.input + te=self.raw_input ) def add_extra_filter(self, filters): @@ -75,3 +84,10 @@ class NewTableEngine: def set_custom_entities(self, entities=None): self.engine.custom_entities = entities return self + + def set_export_file(self): + try: + self.raw_input['e']['d'] = self.input.e.d + return self + except Exception as e: + return False diff --git a/core/utils/te/engines/AbstractEngine.py b/core/utils/te/engines/AbstractEngine.py index 9aafb51..2b242a0 100644 --- a/core/utils/te/engines/AbstractEngine.py +++ b/core/utils/te/engines/AbstractEngine.py @@ -15,6 +15,8 @@ class AbstractEngineClass(abc.ABC): self.options = table_engine.options self.search_cols = table_engine.search_cols self.soft_deleted = table_engine.soft_deleted + self.unselect = table_engine.unselect + self.return_query = table_engine.return_query @abc.abstractmethod def exe(self): diff --git a/core/utils/te/engines/Mongo.py b/core/utils/te/engines/Mongo.py index 20bcba7..9bc1463 100644 --- a/core/utils/te/engines/Mongo.py +++ b/core/utils/te/engines/Mongo.py @@ -1,9 +1,30 @@ import datetime +import uuid from typing import Union, List from uuid import UUID +# import dateutil.parser +from celery import shared_task +# from dateutil.relativedelta import relativedelta + from .AbstractEngine import AbstractEngineClass -from ..utils.Models import FilterItem, FilterScope +from ..utils.Models import FilterItem, FilterScope, OutPutType + + +# def zero_time_data(date, is_max=False): +# _date = dateutil.parser.isoparse(date) +# return datetime.datetime.combine(_date, datetime.time.min if not is_max else datetime.time.max) + + +# def date_year_month_morph(date, is_max=False, has_month=True): +# _date = date +# if len(date) > 7: +# _date = dateutil.parser.isoparse(date) +# if not has_month and len(date) < 5: +# _date = f'{date}-01' +# _date = datetime.datetime.strptime(_date, '%Y-%m') +# _date = _date + relativedelta(day=31) +# return _date if is_max else _date + relativedelta(day=1) class MongoEngine(AbstractEngineClass): @@ -13,15 +34,31 @@ class MongoEngine(AbstractEngineClass): async def exe(self): _f = self.filters = self.set_filters() - tmp = self.query_class.find(_f).sort(**self.set_sort()) + print("_f ====>", _f) + if self.unselect: + tmp = self.query_class.find(_f, self.unselect) + else: + tmp = self.query_class.find(_f) + + sort_by = self.set_sort() + if isinstance(sort_by, dict): + tmp = tmp.sort(**sort_by) + else: + tmp = tmp.sort(sort_by) + self.rows = tmp.limit( self.input.get_limit()).skip(self.input.skip()) + if self.input.e and self.input.e.ot: + self.input.e.d = self.generate_output(self.query_class, _f) + if self.return_query: + return {'query':_f, 'sort':self.set_sort(), 'limit':self.input.get_limit(), 'page':self.input.skip()} return [self.model(**document) async for document in self.rows], await self.count() def query_generator(self): if not self.input.has_filter(): return [] - filters = self.input.filters() + filters = [f for f in self.input.filters() if + (hasattr(f, 'items') and len(f.items)) > 0 or not hasattr(f, 'items')] query = self.generate_scope_filter(filters, general=True) return query @@ -36,7 +73,7 @@ class MongoEngine(AbstractEngineClass): self.input.get_search_cols() and len(self.input.get_search_cols())) else self.search_cols if _search_cols: for col in _search_cols: - items.append({col: {'$regex': self.input.get_text_search()}}) + items.append({col: {'$regex': self.input.get_text_search(), '$options': 'i'}}) filters.append({"$or": items}) if self.input.extra_filter: filters.extend(self.input.extra_filter) @@ -48,13 +85,18 @@ class MongoEngine(AbstractEngineClass): return self.filters def set_sort(self): + if self.input.o.fields: + all_sort = [] + for item in self.input.o.fields: + all_sort.append((item.field, 1 if item.direction == "asc" else -1)) + return all_sort if self.input.o.field: return {'key_or_list': self.input.o.field, 'direction': 1 if self.input.o.direction == "asc" else -1} elif self.default_sort_by: return self.default_sort_by else: - return {'key_or_list': 'created_at', 'direction': -1} + return {'key_or_list': '$natural', 'direction': -1} @staticmethod def filter_item_generator(item: FilterItem) -> dict: @@ -64,13 +106,14 @@ class MongoEngine(AbstractEngineClass): "$elemMatch": { 'field_id': UUID(item.col), 'value': { - **MongoEngine.get_expression_filter(item.op, MongoEngine.cast_value(item.val, item.type), )} + **MongoEngine.get_expression_filter(item.op, MongoEngine.cast_value(item.val, item.type), + field_type=item.type)} } } } return {item.col: { **MongoEngine.get_expression_filter(col=item.col, op=item.op, - val=MongoEngine.cast_value(item.val, item.type), )}} + val=MongoEngine.cast_value(item.val, item.type), field_type=item.type)}} @staticmethod def is_valid_uuid(uuid_to_test, version=4): @@ -80,9 +123,12 @@ class MongoEngine(AbstractEngineClass): return False return str(uuid_obj) == str(uuid_to_test) - @staticmethod def cast_value(val: Union[str, List], type: str) -> any: + if val == [] or type == 'BOOL': + return val + if not val: + return None if type in ['NUMBER', 'STACK', 'FLOAT']: if isinstance(val, List): return [float(x) for x in val] @@ -97,20 +143,37 @@ class MongoEngine(AbstractEngineClass): return UUID(str(val)) return val - @staticmethod - def get_expression_filter(col=None, op: str = 'eq', val: str = '') -> dict: + def get_expression_filter(col=None, op: str = 'eq', val: str = '', field_type: str = None) -> dict: _op = MongoEngine.get_operator(op) + + # Date Type Operator + if field_type in ['DATE', 'DATE_TIME', 'DATETIME', 'DATE_MONTH', 'DATE_YEAR']: + if op == "eq": + return {'$gte': val[0], '$lte': val[1]} + if op == "n_eq": + return {'$lte': val[0], '$gte': val[1]} + if op == "lt": + return {'$lt': val[1]} + if op == "gt": + return {'$gt': val[0]} + if op == "lte": + return {'$lt': val[1]} + if op == "gte": + return {'$gt': val[0]} + if _op == "sw": - return {'$regex': f'^{val}'} + return {'$regex': f'^{val}', '$options': 'i'} elif _op == "ew": - return {'$regex': f'{val}$'} + return {'$regex': f'{val}$', '$options': 'i'} elif _op == "n_ew": - return {'$regex': f'(? 1 if has_join: - - pass + # pass + return self.join_filter(item) else: - return getattr(getattr(self.model, item.col), self.get_operator(item.op))(item.val) + sm = self.model.c if isinstance(self.model, TableValuedAlias) else self.model + # if item.type in ['DATE', 'DATE_TIME']: + # if item.op == 'eq': + # return getattr(sm, item.col).between(zero_time_data(item.val), + # zero_time_data(item.val, is_max=True)) + # if item.op in ['gte', 'gt']: + # item.val = zero_time_data(item.val) + # print(item.val,'GT') + # if item.op in ['lt', 'lte']: + # item.val = zero_time_data(item.val, is_max=True) + # print(item.val,'LT') + + return getattr(getattr(sm, item.col), self.get_operator(item.op))(item.val) def generate_scope_filter(self, filters: list, general=False): if len(filters) == 1: @@ -144,3 +174,74 @@ class MsSqlEngine(AbstractEngineClass): filters = [_op(_a, _b)] + filters return self.generate_scope_filter(filters, general) + + def join_filter(self, item: FilterItem): + inner_joins = item.col.split('.')[-2:] + join = None + for self_join in self.joins: + if inner_joins[0] == self_join.__name__ or self.normalize_model_name(inner_joins[0]) == self_join.__name__: + join = self_join + return getattr(getattr(join, inner_joins[1]), self.get_operator(item.op))(item.val) if join else None + + @staticmethod + def normalize_model_name(obj): + if isinstance(obj, str): + return obj.replace('_', '') + else: + result = {} + model_name = str([i for i in obj.keys()][0]).rsplit('_', 1) + model_name.pop(-1) + model_name = ''.join(model_name) + for index in obj.keys(): + item = index + for i in range(5): + item = item.replace(f'_{i}_', '_') + if item[-1] == '_': + item = item[:-1] + item = item.replace(model_name + '_', '') + result[item] = obj[index] + return result + + def generate_output(self, query, _f): + if self.input.e.ot == OutPutType.EXEL: + tmp = query.filter(_f).order_by(self.set_sort()).set_label_style(LABEL_STYLE_TABLENAME_PLUS_COL) + query = tmp.statement.compile() + params = query.params + file_name = str(uuid.uuid4().hex) + '.xlsx' + MsSqlEngine.output_generator(str(query).replace('false', '0'), params, + str(self.query_class.bind.url), file_name=file_name, cols=self.input.e.c) + return file_name + elif self.input.e.ot == OutPutType.JSON: + pass + elif self.input.e.ot == OutPutType.PDF: + pass + else: + raise Exception("Output type not supported.") + + @staticmethod + @shared_task(name='output_generator_task') + def output_generator(query, params, url, file_name, cols): + # engine = create_engine(url=url) + # session = sessionmaker(autocommit=False, autoflush=False, bind=engine) + # db = session() + # datas = db.execute(query, params) + # file_dir = os.path.join(tempfile.gettempdir(), file_name) + # head_cols = list(str(string.ascii_uppercase)) + # headers = cols.keys() + # # workbook = xlsxwriter.Workbook(file_dir) + # worksheet = workbook.add_worksheet() + # for header in headers: + # worksheet.write(f'{head_cols.pop(0)}1', cols[header]) + # row = 1 + # for item in datas: + # row += 1 + # head_cols = list(str(string.ascii_uppercase)) + # data = MsSqlEngine.normalize_model_name(item) + # for col in headers: + # if col[-1] == '_': + # col = col[:-1] + # col = col.replace('.', '_') + # worksheet.write(f'{head_cols.pop(0)}{row}', data[col] if col in data else '-') + # workbook.close() + ... + return True diff --git a/core/utils/te/utils/Models.py b/core/utils/te/utils/Models.py index 0e0e10d..49f24f1 100644 --- a/core/utils/te/utils/Models.py +++ b/core/utils/te/utils/Models.py @@ -1,8 +1,10 @@ from enum import Enum -from typing import Optional, Any, List +from typing import Any +from typing import Generic, TypeVar, List, Optional import pydantic from pydantic import BaseModel +from pydantic.generics import GenericModel class FilterItem(BaseModel): @@ -12,6 +14,7 @@ class FilterItem(BaseModel): op: Optional[str] type: Optional[str] val: Optional[Any] + hidden: Optional[bool] @staticmethod def ValidItem(item: dict): @@ -30,6 +33,12 @@ class Cr(str, Enum): AND = 'and' +class OutPutType(str, Enum): + JSON = 'json' + EXEL = 'exel' + PDF = 'pdf' + + class FilterScope(BaseModel): id: str items: Optional[List[FilterItem]] @@ -60,6 +69,7 @@ class FilterScope(BaseModel): class Page(BaseModel): c: int = 1 s: int = 5 + force: Optional[bool] = False class SearchModel(BaseModel): @@ -67,11 +77,23 @@ class SearchModel(BaseModel): cols: Optional[List[Any]] +class ExportModel(BaseModel): + ot: Optional[OutPutType] + c: Optional[dict] + d: Optional[str] + + class SortModel(BaseModel): field: Optional[str] direction: Optional[str] +class MultiSortModel(BaseModel): + field: Optional[str] + direction: Optional[str] + fields: Optional[List[SortModel]] + + class JoinModel(BaseModel): model: BaseModel show: List[str] @@ -88,17 +110,19 @@ class Config: class TableEngineInput(BaseModel): f: Optional[List[FilterScope]] extra_filter: list = [] - o: Optional[SortModel] + o: Optional[MultiSortModel] p: Optional[Page] = Page() s: Optional[SearchModel] + e: Optional[ExportModel] is_deleted: Optional[bool] def __init__(self, **kwargs): super().__init__(**kwargs) _input = {**{'p': {'c': 1, 's': 5}, - 'f': [], 's': {}, 'o': {}}, **(kwargs or {})} + 'f': [], 's': {}, 'o': {}, 'e': {}}, **(kwargs or {})} self.p = Page(**_input.get('p')) - self.o = SortModel(**_input.get('o')) + self.o = MultiSortModel(**_input.get('o')) + self.e = ExportModel(**_input.get('e')) if _input.get('f'): self.f = [FilterScope(**f) for f in filter(lambda fs: FilterScope.ValidScope(fs), _input.get('f'))] @@ -135,7 +159,10 @@ class TableEngineInput(BaseModel): return self.page() + 1 def get_limit(self): + # HIT : LIMIT 0 : in Mongo Return all records limit = self.p.s if self.p else 20 + if self.p.force: + return 1000000 return limit if limit < 100 else 100 def text_search_cols(self, cols: List[str]): @@ -158,3 +185,26 @@ class TableEngineInput(BaseModel): class WorkingMode(Enum): Development = "development" Production = "production" + + +T = TypeVar('T') + + +class TableEngineInputType(GenericModel, Generic[T]): + f: Optional[List] + o: Optional[dict] = {} + p: Optional[dict] = {} + s: Optional[dict] = {} + e: Optional[dict] = {} + + +class TableEngineExport(GenericModel, Generic[T]): + rows: Optional[List[T]] + total: Optional[int] + page: Optional[int] + te: Optional[TableEngineInputType] + + +class TableEngineRequestType(GenericModel, Generic[T]): + te: Optional[TableEngineInputType] + extra: Optional[dict]