feat(project):change mongo

core/utils/te/__init__.py

@@ -1,14 +1,16 @@
 from typing import List
 
+from starlette.responses import FileResponse
+
 from .engines import MsSqlEngine, MongoEngine, PgSqlEngine, AbstractEngine
 from .utils.Models import TableEngineInput
 from .utils.Models import WorkingMode, JoinModel
-from ..table_engine import TableEngineExport
+from .utils.Models import TableEngineExport
 
 
 class NewTableEngine:
     def __init__(self, query_class, model, inputs, mode, engine: AbstractEngine, extra_filter,
-                 default_sort_by, joins, search_cols, options=None, soft_deleted=False):
+                 default_sort_by, joins, search_cols, options=None, soft_deleted=False, unselect={}, return_query=False):
         self.model = model
         self.query_class = query_class
         self.mode = mode
@@ -18,12 +20,15 @@ class NewTableEngine:
             self.input = TableEngineInput(**inputs if inputs else {})
         else:
             self.input = TableEngineInput(**inputs.dict() if inputs else {})
+
         self.input.extra_filter = extra_filter
         self.default_sort_by = default_sort_by
         self.joins = joins
         self.soft_deleted = soft_deleted
         self.options = options
         self.search_cols = search_cols
+        self.unselect = unselect
+        self.return_query = return_query
         self.engine = engine(self)
 
     @staticmethod
@@ -33,10 +38,10 @@ class NewTableEngine:
               mode: WorkingMode = WorkingMode.Development,
               extra_filter=None,
               soft_deleted=False,
-              default_sort_by=None, joins=None, search_cols=[]):
+              default_sort_by=None, joins=None, search_cols=[] ,unselect={},  return_query=False):
         return NewTableEngine(query_class, model, inputs, mode, engine=MongoEngine, extra_filter=extra_filter,
                               default_sort_by=default_sort_by, joins=joins, search_cols=search_cols,
-                              soft_deleted=soft_deleted)
+                              soft_deleted=soft_deleted,unselect=unselect,  return_query=return_query)
 
     @staticmethod
     def mssql(query_class, model, inputs, mode: WorkingMode = WorkingMode.Development, extra_filter=None,
@@ -53,15 +58,19 @@ class NewTableEngine:
     async def exe(self):
         return await self.engine.exe()
 
-    async def to_dict(self) -> TableEngineExport:
+    async def to_dict(self):
+        if self.return_query:
+            query = await self.exe()
+            return query
         rows, count = await self.exe()
         self.input.extra_filter = None
         del self.engine
+        self.set_export_file()
         return TableEngineExport(
             rows=rows,
             page=self.input.virtual_page(),
             total=count,
-            te=self.input
+            te=self.raw_input
         )
 
     def add_extra_filter(self, filters):
@@ -75,3 +84,10 @@ class NewTableEngine:
     def set_custom_entities(self, entities=None):
         self.engine.custom_entities = entities
         return self
+
+    def set_export_file(self):
+        try:
+            self.raw_input['e']['d'] = self.input.e.d
+            return self
+        except Exception as e:
+            return False

core/utils/te/engines/AbstractEngine.py

@@ -15,6 +15,8 @@ class AbstractEngineClass(abc.ABC):
         self.options = table_engine.options
         self.search_cols = table_engine.search_cols
         self.soft_deleted = table_engine.soft_deleted
+        self.unselect = table_engine.unselect
+        self.return_query = table_engine.return_query
 
     @abc.abstractmethod
     def exe(self):

core/utils/te/engines/Mongo.py

@@ -1,9 +1,30 @@
 import datetime
+import uuid
 from typing import Union, List
 from uuid import UUID
 
+# import dateutil.parser
+from celery import shared_task
+# from dateutil.relativedelta import relativedelta
+
 from .AbstractEngine import AbstractEngineClass
-from ..utils.Models import FilterItem, FilterScope
+from ..utils.Models import FilterItem, FilterScope, OutPutType
+
+
+# def zero_time_data(date, is_max=False):
+#     _date = dateutil.parser.isoparse(date)
+#     return datetime.datetime.combine(_date, datetime.time.min if not is_max else datetime.time.max)
+
+
+# def date_year_month_morph(date, is_max=False, has_month=True):
+#     _date = date
+#     if len(date) > 7:
+#         _date = dateutil.parser.isoparse(date)
+#     if not has_month and len(date) < 5:
+#         _date = f'{date}-01'
+#     _date = datetime.datetime.strptime(_date, '%Y-%m')
+#     _date = _date + relativedelta(day=31)
+#     return _date if is_max else _date + relativedelta(day=1)
 
 
 class MongoEngine(AbstractEngineClass):
@@ -13,15 +34,31 @@ class MongoEngine(AbstractEngineClass):
 
     async def exe(self):
         _f = self.filters = self.set_filters()
-        tmp = self.query_class.find(_f).sort(**self.set_sort())
+        print("_f  ====>", _f)
+        if self.unselect:
+            tmp = self.query_class.find(_f, self.unselect)
+        else:
+            tmp = self.query_class.find(_f)
+
+        sort_by = self.set_sort()
+        if isinstance(sort_by, dict):
+            tmp = tmp.sort(**sort_by)
+        else:
+            tmp = tmp.sort(sort_by)
+
         self.rows = tmp.limit(
             self.input.get_limit()).skip(self.input.skip())
+        if self.input.e and self.input.e.ot:
+            self.input.e.d = self.generate_output(self.query_class, _f)
+        if self.return_query:
+            return {'query':_f, 'sort':self.set_sort(), 'limit':self.input.get_limit(), 'page':self.input.skip()}
         return [self.model(**document) async for document in self.rows], await self.count()
 
     def query_generator(self):
         if not self.input.has_filter():
             return []
-        filters = self.input.filters()
+        filters = [f for f in self.input.filters() if
+                   (hasattr(f, 'items') and len(f.items)) > 0 or not hasattr(f, 'items')]
         query = self.generate_scope_filter(filters, general=True)
         return query
 
@@ -36,7 +73,7 @@ class MongoEngine(AbstractEngineClass):
                     self.input.get_search_cols() and len(self.input.get_search_cols())) else self.search_cols
             if _search_cols:
                 for col in _search_cols:
-                    items.append({col: {'$regex': self.input.get_text_search()}})
+                    items.append({col: {'$regex': self.input.get_text_search(), '$options': 'i'}})
                 filters.append({"$or": items})
         if self.input.extra_filter:
             filters.extend(self.input.extra_filter)
@@ -48,13 +85,18 @@ class MongoEngine(AbstractEngineClass):
         return self.filters
 
     def set_sort(self):
+        if self.input.o.fields:
+            all_sort = []
+            for item in self.input.o.fields:
+                all_sort.append((item.field, 1 if item.direction == "asc" else -1))
+            return all_sort
         if self.input.o.field:
             return {'key_or_list': self.input.o.field,
                     'direction': 1 if self.input.o.direction == "asc" else -1}
         elif self.default_sort_by:
             return self.default_sort_by
         else:
-            return {'key_or_list': 'created_at', 'direction': -1}
+            return {'key_or_list': '$natural', 'direction': -1}
 
     @staticmethod
     def filter_item_generator(item: FilterItem) -> dict:
@@ -64,13 +106,14 @@ class MongoEngine(AbstractEngineClass):
                     "$elemMatch": {
                         'field_id': UUID(item.col),
                         'value': {
-                            **MongoEngine.get_expression_filter(item.op, MongoEngine.cast_value(item.val, item.type), )}
+                            **MongoEngine.get_expression_filter(item.op, MongoEngine.cast_value(item.val, item.type),
+                                                                field_type=item.type)}
                     }
                 }
             }
         return {item.col: {
             **MongoEngine.get_expression_filter(col=item.col, op=item.op,
-                                                val=MongoEngine.cast_value(item.val, item.type), )}}
+                                                val=MongoEngine.cast_value(item.val, item.type), field_type=item.type)}}
 
     @staticmethod
     def is_valid_uuid(uuid_to_test, version=4):
@@ -80,9 +123,12 @@ class MongoEngine(AbstractEngineClass):
             return False
         return str(uuid_obj) == str(uuid_to_test)
 
-
     @staticmethod
     def cast_value(val: Union[str, List], type: str) -> any:
+        if val == [] or type == 'BOOL':
+            return val
+        if not val:
+            return None
         if type in ['NUMBER', 'STACK', 'FLOAT']:
             if isinstance(val, List):
                 return [float(x) for x in val]
@@ -97,20 +143,37 @@ class MongoEngine(AbstractEngineClass):
             return UUID(str(val))
         return val
 
-
     @staticmethod
-    def get_expression_filter(col=None, op: str = 'eq', val: str = '') -> dict:
+    def get_expression_filter(col=None, op: str = 'eq', val: str = '', field_type: str = None) -> dict:
         _op = MongoEngine.get_operator(op)
+
+        # Date Type Operator
+        if field_type in ['DATE', 'DATE_TIME', 'DATETIME', 'DATE_MONTH', 'DATE_YEAR']:
+            if op == "eq":
+                return {'$gte': val[0], '$lte': val[1]}
+            if op == "n_eq":
+                return {'$lte': val[0], '$gte': val[1]}
+            if op == "lt":
+                return {'$lt': val[1]}
+            if op == "gt":
+                return {'$gt': val[0]}
+            if op == "lte":
+                return {'$lt': val[1]}
+            if op == "gte":
+                return {'$gt': val[0]}
+
         if _op == "sw":
-            return {'$regex': f'^{val}'}
+            return {'$regex': f'^{val}', '$options': 'i'}
         elif _op == "ew":
-            return {'$regex': f'{val}$'}
+            return {'$regex': f'{val}$', '$options': 'i'}
         elif _op == "n_ew":
-            return {'$regex': f'(?<!{val})$'}
+            return {'$regex': f'(?<!{val})$', '$options': 'i'}
         elif _op == "n_sw":
-            return {'$regex': f'^(?!{val})'}
+            return {'$regex': f'^(?!{val})', '$options': 'i'}
         elif _op == "n_like":
-            return {'$not': {'$regex': val}}
+            return {'$not': {'$regex': val, '$options': 'i'}}
+        elif _op == "like":
+            return {'$regex': val, '$options': 'i'}
         elif _op == "bet":
             if isinstance(val, str):
                 return {'$gte': val.split('-')[0], '$lte': val.split('-')[1]}
@@ -124,9 +187,9 @@ class MongoEngine(AbstractEngineClass):
         elif _op == "n_exi":
             return {'$exists': False}
         elif _op == "in":
-            return {'$exists': True}
+            return {'$in': val}
         elif _op == "not_in":
-            return {'$exists': False}
+            return {'$nin': val}
         return {_op: val}
 
     @staticmethod
@@ -138,7 +201,7 @@ class MongoEngine(AbstractEngineClass):
             "ge": "$ge",
             "lt": "$lt",
             "le": "$le",
-            "like": "$regex",
+            "like": "like",
             "n_like": "n_like",
             "sw": "sw",
             "ew": "ew",
@@ -185,3 +248,24 @@ class MongoEngine(AbstractEngineClass):
 
     def set_joins(self):
         pass
+
+    def generate_output(self, query, _f):
+        print(_f.__dict__)
+        return ''
+        if self.input.e.ot == OutPutType.EXEL:
+            file_name = str(uuid.uuid4().hex) + '.xlsx'
+            return _f.dict()
+            MongoEngine.output_generator(str(query).replace('false', '0'), params,
+                                         str(self.query_class.bind.url), file_name=file_name, cols=self.input.e.c)
+            return file_name
+        elif self.input.e.ot == OutPutType.JSON:
+            pass
+        elif self.input.e.ot == OutPutType.PDF:
+            pass
+        else:
+            raise Exception("Output type not supported.")
+
+    @staticmethod
+    @shared_task(name='output_generator_task_mongo')
+    def output_generator(query, params, url, file_name, cols):
+        pass

core/utils/te/engines/Mssql.py

@@ -1,8 +1,18 @@
-from sqlalchemy import or_, and_, desc, asc
+import os
+import string
+import tempfile
+import uuid
+
+# import xlsxwriter
+from celery import shared_task
+from sqlalchemy import or_, and_, desc, asc, create_engine, LABEL_STYLE_TABLENAME_PLUS_COL
+from sqlalchemy.orm import sessionmaker
 from sqlalchemy.sql.elements import BooleanClauseList, BinaryExpression, UnaryExpression
+from sqlalchemy.sql.selectable import TableValuedAlias
 
 from .AbstractEngine import AbstractEngineClass
-from ..utils.Models import FilterItem, FilterScope
+# from .Mongo import zero_time_data
+from ..utils.Models import FilterItem, FilterScope, OutPutType
 
 
 class MsSqlEngine(AbstractEngineClass):
@@ -16,7 +26,8 @@ class MsSqlEngine(AbstractEngineClass):
 
     async def exe(self):
         _f = self.set_filters()
-        if not self.custom_entities:
+        print(_f)
+        if self.custom_entities is not None:
             query = self.query_class.query(self.model)
         else:
             query = self.query_class.query(*self.custom_entities)
@@ -29,9 +40,11 @@ class MsSqlEngine(AbstractEngineClass):
 
         if self.options:
             query = query.options(*self.options)
+        if self.input.e and self.input.e.ot:
+            self.input.e.d = self.generate_output(query, _f)
         tmp = query.filter(_f).order_by(self.set_sort())
         self.rows = tmp.limit(self.input.get_limit()).offset(self.input.skip()).all()
-        return self.rows, tmp.count(), tmp
+        return self.rows, tmp.count()
 
     def query_generator(self):
         if not self.input.has_filter():
@@ -42,8 +55,9 @@ class MsSqlEngine(AbstractEngineClass):
 
     def set_sort(self):
         if self.input.o.field:
-            self.sort = asc(getattr(self.model, self.input.o.field)) if self.input.o.direction == 'asc' else desc(
+            sm = self.model.c if isinstance(self.model, TableValuedAlias) else self.model
-                getattr(self.model, self.input.o.field))
+            self.sort = asc(getattr(sm, self.input.o.field)) if self.input.o.direction == 'asc' else desc(
+                getattr(sm, self.input.o.field))
             return self.sort
         elif isinstance(self.default_sort_by, UnaryExpression):
             return self.default_sort_by
@@ -65,9 +79,13 @@ class MsSqlEngine(AbstractEngineClass):
                     if has_join:
                         pass
                     else:
+                        sm = self.model.c if isinstance(self.model, TableValuedAlias) else self.model
                         items.append(
-                            getattr(self.model, col).ilike("%{search}%".format(search=self.input.get_text_search())))
+                            getattr(sm, col).ilike(
+                                "%{search}%".format(search=self.input.get_text_search())))
+
                 filters.append(or_(*items))
+
         if self.input.extra_filter:
             for f in self.input.extra_filter:
                 filters.append(f)
@@ -88,7 +106,7 @@ class MsSqlEngine(AbstractEngineClass):
             "n_eq": "__ne__",
             "gt": "__gt__",
             "gte": "__ge__",
-            "lte": "__lte__",
+            "lte": "__le__",
             "lt": "__lt__",
             "like": "ilike",
             "n_like": "notlike",
@@ -115,10 +133,22 @@ class MsSqlEngine(AbstractEngineClass):
     def filter_item_generator(self, item: FilterItem):
         has_join = len(item.col.split('.')) > 1
         if has_join:
-
+            # pass
-            pass
+            return self.join_filter(item)
         else:
-            return getattr(getattr(self.model, item.col), self.get_operator(item.op))(item.val)
+            sm = self.model.c if isinstance(self.model, TableValuedAlias) else self.model
+            # if item.type in ['DATE', 'DATE_TIME']:
+            #     if item.op == 'eq':
+            #         return getattr(sm, item.col).between(zero_time_data(item.val),
+            #                                              zero_time_data(item.val, is_max=True))
+            #     if item.op in ['gte', 'gt']:
+            #         item.val = zero_time_data(item.val)
+            #         print(item.val,'GT')
+            #     if item.op in ['lt', 'lte']:
+            #         item.val = zero_time_data(item.val, is_max=True)
+            #         print(item.val,'LT')
+
+            return getattr(getattr(sm, item.col), self.get_operator(item.op))(item.val)
 
     def generate_scope_filter(self, filters: list, general=False):
         if len(filters) == 1:
@@ -144,3 +174,74 @@ class MsSqlEngine(AbstractEngineClass):
 
             filters = [_op(_a, _b)] + filters
             return self.generate_scope_filter(filters, general)
+
+    def join_filter(self, item: FilterItem):
+        inner_joins = item.col.split('.')[-2:]
+        join = None
+        for self_join in self.joins:
+            if inner_joins[0] == self_join.__name__ or self.normalize_model_name(inner_joins[0]) == self_join.__name__:
+                join = self_join
+        return getattr(getattr(join, inner_joins[1]), self.get_operator(item.op))(item.val) if join else None
+
+    @staticmethod
+    def normalize_model_name(obj):
+        if isinstance(obj, str):
+            return obj.replace('_', '')
+        else:
+            result = {}
+            model_name = str([i for i in obj.keys()][0]).rsplit('_', 1)
+            model_name.pop(-1)
+            model_name = ''.join(model_name)
+            for index in obj.keys():
+                item = index
+                for i in range(5):
+                    item = item.replace(f'_{i}_', '_')
+                if item[-1] == '_':
+                    item = item[:-1]
+                item = item.replace(model_name + '_', '')
+                result[item] = obj[index]
+            return result
+
+    def generate_output(self, query, _f):
+        if self.input.e.ot == OutPutType.EXEL:
+            tmp = query.filter(_f).order_by(self.set_sort()).set_label_style(LABEL_STYLE_TABLENAME_PLUS_COL)
+            query = tmp.statement.compile()
+            params = query.params
+            file_name = str(uuid.uuid4().hex) + '.xlsx'
+            MsSqlEngine.output_generator(str(query).replace('false', '0'), params,
+                                         str(self.query_class.bind.url), file_name=file_name, cols=self.input.e.c)
+            return file_name
+        elif self.input.e.ot == OutPutType.JSON:
+            pass
+        elif self.input.e.ot == OutPutType.PDF:
+            pass
+        else:
+            raise Exception("Output type not supported.")
+
+    @staticmethod
+    @shared_task(name='output_generator_task')
+    def output_generator(query, params, url, file_name, cols):
+        # engine = create_engine(url=url)
+        # session = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+        # db = session()
+        # datas = db.execute(query, params)
+        # file_dir = os.path.join(tempfile.gettempdir(), file_name)
+        # head_cols = list(str(string.ascii_uppercase))
+        # headers = cols.keys()
+        # # workbook = xlsxwriter.Workbook(file_dir)
+        # worksheet = workbook.add_worksheet()
+        # for header in headers:
+        #     worksheet.write(f'{head_cols.pop(0)}1', cols[header])
+        # row = 1
+        # for item in datas:
+        #     row += 1
+        #     head_cols = list(str(string.ascii_uppercase))
+        #     data = MsSqlEngine.normalize_model_name(item)
+        #     for col in headers:
+        #         if col[-1] == '_':
+        #             col = col[:-1]
+        #         col = col.replace('.', '_')
+        #         worksheet.write(f'{head_cols.pop(0)}{row}', data[col] if col in data else '-')
+        # workbook.close()
+        ...
+        return True

core/utils/te/utils/Models.py

@@ -1,8 +1,10 @@
 from enum import Enum
-from typing import Optional, Any, List
+from typing import Any
+from typing import Generic, TypeVar, List, Optional
 
 import pydantic
 from pydantic import BaseModel
+from pydantic.generics import GenericModel
 
 
 class FilterItem(BaseModel):
@@ -12,6 +14,7 @@ class FilterItem(BaseModel):
     op: Optional[str]
     type: Optional[str]
     val: Optional[Any]
+    hidden: Optional[bool]
 
     @staticmethod
     def ValidItem(item: dict):
@@ -30,6 +33,12 @@ class Cr(str, Enum):
     AND = 'and'
 
 
+class OutPutType(str, Enum):
+    JSON = 'json'
+    EXEL = 'exel'
+    PDF = 'pdf'
+
+
 class FilterScope(BaseModel):
     id: str
     items: Optional[List[FilterItem]]
@@ -60,6 +69,7 @@ class FilterScope(BaseModel):
 class Page(BaseModel):
     c: int = 1
     s: int = 5
+    force: Optional[bool] = False
 
 
 class SearchModel(BaseModel):
@@ -67,11 +77,23 @@ class SearchModel(BaseModel):
     cols: Optional[List[Any]]
 
 
+class ExportModel(BaseModel):
+    ot: Optional[OutPutType]
+    c: Optional[dict]
+    d: Optional[str]
+
+
 class SortModel(BaseModel):
     field: Optional[str]
     direction: Optional[str]
 
 
+class MultiSortModel(BaseModel):
+    field: Optional[str]
+    direction: Optional[str]
+    fields: Optional[List[SortModel]]
+
+
 class JoinModel(BaseModel):
     model: BaseModel
     show: List[str]
@@ -88,17 +110,19 @@ class Config:
 class TableEngineInput(BaseModel):
     f: Optional[List[FilterScope]]
     extra_filter: list = []
-    o: Optional[SortModel]
+    o: Optional[MultiSortModel]
     p: Optional[Page] = Page()
     s: Optional[SearchModel]
+    e: Optional[ExportModel]
     is_deleted: Optional[bool]
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
         _input = {**{'p': {'c': 1, 's': 5},
-                     'f': [], 's': {}, 'o': {}}, **(kwargs or {})}
+                     'f': [], 's': {}, 'o': {}, 'e': {}}, **(kwargs or {})}
         self.p = Page(**_input.get('p'))
-        self.o = SortModel(**_input.get('o'))
+        self.o = MultiSortModel(**_input.get('o'))
+        self.e = ExportModel(**_input.get('e'))
         if _input.get('f'):
             self.f = [FilterScope(**f) for f in
                       filter(lambda fs: FilterScope.ValidScope(fs), _input.get('f'))]
@@ -135,7 +159,10 @@ class TableEngineInput(BaseModel):
         return self.page() + 1
 
     def get_limit(self):
+        # HIT  : LIMIT 0 : in Mongo Return all records
         limit = self.p.s if self.p else 20
+        if self.p.force:
+            return 1000000
         return limit if limit < 100 else 100
 
     def text_search_cols(self, cols: List[str]):
@@ -158,3 +185,26 @@ class TableEngineInput(BaseModel):
 class WorkingMode(Enum):
     Development = "development"
     Production = "production"
+
+
+T = TypeVar('T')
+
+
+class TableEngineInputType(GenericModel, Generic[T]):
+    f: Optional[List]
+    o: Optional[dict] = {}
+    p: Optional[dict] = {}
+    s: Optional[dict] = {}
+    e: Optional[dict] = {}
+
+
+class TableEngineExport(GenericModel, Generic[T]):
+    rows: Optional[List[T]]
+    total: Optional[int]
+    page: Optional[int]
+    te: Optional[TableEngineInputType]
+
+
+class TableEngineRequestType(GenericModel, Generic[T]):
+    te: Optional[TableEngineInputType]
+    extra: Optional[dict]