KaneAI - World’s First E2E Software Testing Agent.Book a Demo Join TestMu Conference

How to use generate_output method in Kiwi

Best Python code snippet using Kiwi_python

preprocess_train_dev_data.py

Source:preprocess_train_dev_data.py

...86        self.sql = sql87        self.question = question88        self.history = history89        self.keywords = ('intersect', 'except', 'union')90    def generate_output(self):91        for key in self.sql:92            if key in self.keywords and self.sql[key]:93                return self.history + ['root'], key, self.sql[key]94        return self.history + ['root'], 'none', self.sql95class KeyWordPredictor:96    def __init__(self, question, sql, history):97        self.sql = sql98        self.question = question99        self.history = history100        self.keywords = ('select', 'where', 'groupBy', 'orderBy', 'limit', 'having')101    def generate_output(self):102        sql_keywords = []103        for key in self.sql:104            if key in self.keywords and self.sql[key]:105                sql_keywords.append(key)106        return self.history, [len(sql_keywords), sql_keywords], self.sql107class ColPredictor:108    def __init__(self, question, sql, table, history,kw=None):109        self.sql = sql110        self.question = question111        self.history = history112        self.table = table113        self.keywords = ('select', 'where', 'groupBy', 'orderBy', 'having')114        self.kw = kw115    def generate_output(self):116        ret = []117        candidate_keys = self.sql.keys()118        if self.kw:119            candidate_keys = [self.kw]120        for key in candidate_keys:121            if key in self.keywords and self.sql[key]:122                cols = []123                sqls = []124                if key == 'groupBy':125                    sql_cols = self.sql[key]126                    for col in sql_cols:127                        cols.append((index_to_column_name(col[1], self.table), col[2]))128                        sqls.append(col)129                elif key == 'orderBy':130                    sql_cols = self.sql[key][1]131                    for col in sql_cols:132                        cols.append((index_to_column_name(col[1][1], self.table), col[1][2]))133                        sqls.append(col)134                elif key == 'select':135                    sql_cols = self.sql[key][1]136                    for col in sql_cols:137                        cols.append((index_to_column_name(col[1][1][1], self.table), col[1][1][2]))138                        sqls.append(col)139                elif key == 'where' or key == 'having':140                    sql_cols = self.sql[key]141                    for col in sql_cols:142                        if not isinstance(col, list):143                            continue144                        try:145                            cols.append((index_to_column_name(col[2][1][1], self.table), col[2][1][2]))146                        except:147                            print("Key:{} Col:{} Question:{}".format(key, col, self.question))148                        sqls.append(col)149                ret.append((150                    self.history + [key], (len(cols), cols), sqls151                ))152        return ret153        # ret.append(history+[key],)154class OpPredictor:155    def __init__(self, question, sql, history):156        self.sql = sql157        self.question = question158        self.history = history159        # self.keywords = ('select', 'where', 'groupBy', 'orderBy', 'having')160    def generate_output(self):161        return self.history, convert_to_op_index(self.sql[0],self.sql[1]), (self.sql[3], self.sql[4])162class AggPredictor:163    def __init__(self, question, sql, history,kw=None):164        self.sql = sql165        self.question = question166        self.history = history167        self.kw = kw168    def generate_output(self):169        label = -1170        if self.kw:171            key = self.kw172        else:173            key = self.history[-2]174        if key == 'select':175            label = self.sql[0]176        elif key == 'orderBy':177            label = self.sql[1][0]178        elif key == 'having':179            label = self.sql[2][1][0]180        return self.history, label181# class RootTemPredictor:182#     def __init__(self, question, sql):183#         self.sql = sql184#         self.question = question185#         self.keywords = ('intersect', 'except', 'union')186#187#     def generate_output(self):188#         for key in self.sql:189#             if key in self.keywords:190#                 return ['ROOT'], key, self.sql[key]191#         return ['ROOT'], 'none', self.sql192class DesAscPredictor:193    def __init__(self, question, sql, table, history):194        self.sql = sql195        self.question = question196        self.history = history197        self.table = table198    def generate_output(self):199        for key in self.sql:200            if key == "orderBy" and self.sql[key]:201                # self.history.append(key)202                try:203                    col = self.sql[key][1][0][1][1]204                except:205                    print("question:{} sql:{}".format(self.question, self.sql))206                # self.history.append(index_to_column_name(col, self.table))207                # self.history.append(self.sql[key][1][0][1][0])208                if self.sql[key][0] == "asc" and self.sql["limit"]:209                    label = 0210                elif self.sql[key][0] == "asc" and not self.sql["limit"]:211                    label = 1212                elif self.sql[key][0] == "desc" and self.sql["limit"]:213                    label = 2214                else:215                    label = 3216                return self.history+[index_to_column_name(col, self.table),self.sql[key][1][0][1][0]], label217class AndOrPredictor:218    def __init__(self, question, sql, table, history):219        self.sql = sql220        self.question = question221        self.history = history222        self.table = table223    def generate_output(self):224        if 'where' in self.sql and self.sql['where'] and len(self.sql['where']) > 1:225            return self.history,COND_OPS[self.sql['where'][1]]226        return self.history,-1227def parser_item_with_long_history(question_tokens, sql, table, history, dataset):228    table_schema = [229        table["table_names"],230        table["column_names"],231        table["column_types"]232    ]233    stack = [("root",sql)]234    with_join = False235    fk_dict = defaultdict(list)236    for fk in table["foreign_keys"]:237        fk_dict[fk[0]].append(fk[1])238        fk_dict[fk[1]].append(fk[0])239    while len(stack) > 0:240        node = stack.pop()241        if node[0] == "root":242            history, label, sql = MultiSqlPredictor(question_tokens, node[1], history).generate_output()243            dataset['multi_sql_dataset'].append({244                "question_tokens": question_tokens,245                "ts": table_schema,246                "history": history[:],247                "label": SQL_OPS[label]248            })249            history.append(label)250            if label == "none":251                stack.append((label,sql))252            else:253                node[1][label] = None254                stack.append((label, node[1],sql))255            # if label != "none":256                # stack.append(("none",node[1]))257        elif node[0] in ('intersect', 'except', 'union'):258            stack.append(("root",node[1]))259            stack.append(("root",node[2]))260        elif node[0] == "none":261            with_join = len(node[1]["from"]["table_units"]) > 1262            history, label, sql = KeyWordPredictor(question_tokens, node[1], history).generate_output()263            label_idxs = []264            for item in label[1]:265                if item in KW_DICT:266                    label_idxs.append(KW_DICT[item])267            label_idxs.sort()268            dataset['keyword_dataset'].append({269                "question_tokens": question_tokens,270                "ts": table_schema,271                "history": history[:],272                "label": label_idxs273            })274            if "having" in label[1]:275                stack.append(("having",node[1]))276            if "orderBy" in label[1]:277                stack.append(("orderBy",node[1]))278            if "groupBy" in label[1]:279                if "having" in label[1]:280                    dataset['having_dataset'].append({281                        "question_tokens": question_tokens,282                        "ts": table_schema,283                        "history": history[:],284                        "gt_col":node[1]["groupBy"][0][1],285                        "label": 1286                    })287                else:288                    dataset['having_dataset'].append({289                        "question_tokens": question_tokens,290                        "ts": table_schema,291                        "history": history[:],292                        "gt_col":node[1]["groupBy"][0][1],293                        "label": 0294                    })295                stack.append(("groupBy",node[1]))296            if "where" in label[1]:297                stack.append(("where",node[1]))298            if "select" in label[1]:299                stack.append(("select",node[1]))300        elif node[0] in ("select","having","orderBy"):301            # if node[0] != "orderBy":302            history.append(node[0])303            if node[0] == "orderBy":304                orderby_ret = DesAscPredictor(question_tokens, node[1], table, history).generate_output()305                if orderby_ret:306                    dataset['des_asc_dataset'].append({307                        "question_tokens": question_tokens,308                        "ts": table_schema,309                        "history": orderby_ret[0],310                        "gt_col":node[1]["orderBy"][1][0][1][1],311                        "label": orderby_ret[1]312                    })313                    # history.append(orderby_ret[1])314            col_ret = ColPredictor(question_tokens, node[1], table, history,node[0]).generate_output()315            agg_col_dict = dict()316            op_col_dict = dict()317            for h, l, s in col_ret:318                if l[0] == 0:319                    print("Warning: predicted 0 columns!")320                    continue321                dataset['col_dataset'].append({322                    "question_tokens": question_tokens,323                    "ts": table_schema,324                    "history": history[:],325                    "label":get_label_cols(with_join,fk_dict,l[1])326                })327                for col, sql_item in zip(l[1], s):328                    key = "{}{}{}".format(col[0][0],col[0][1],col[0][2])329                    if key not in agg_col_dict:330                        agg_col_dict[key] = [(sql_item,col[0])]331                    else:332                        agg_col_dict[key].append((sql_item,col[0]))333                    if key not in op_col_dict:334                        op_col_dict[key] = [(sql_item,col[0])]335                    else:336                        op_col_dict[key].append((sql_item,col[0]))337                for key in agg_col_dict:338                    stack.append(("col",node[0],agg_col_dict[key],op_col_dict[key]))339        elif node[0] == "col":340            history.append(node[2][0][1])341            if node[1] == "where":342                stack.append(("op",node[2],"where"))343            else:344                labels = []345                for sql_item,col in node[2]:346                    _, label = AggPredictor(question_tokens, sql_item, history,node[1]).generate_output()347                    if label-1 >= 0:348                        labels.append(label-1)349                # print(node[2][0][1][2])350                dataset['agg_dataset'].append({351                    "question_tokens": question_tokens,352                    "ts": table_schema,353                    "history": history[:],354                    "gt_col":node[2][0][1][2],355                    "label": labels[:min(len(labels),3)]356                })357                if node[1] == "having":358                    stack.append(("op", node[2], "having"))359                # if len(labels) == 0:360                #     history.append("none")361                # else:362                if len(labels) > 0:363                    history.append(AGG_OPS[labels[0]+1])364        elif node[0] == "op":365            # history.append(node[1][0][1])366            labels = []367            # if len(labels) > 2:368            #     print(question_tokens)369            dataset['op_dataset'].append({370                "question_tokens": question_tokens,371                "ts": table_schema,372                "history": history[:],373                "gt_col": node[1][0][1][2],374                "label": labels375            })376            for sql_item,col in node[1]:377                _, label, s = OpPredictor(question_tokens, sql_item, history).generate_output()378                if label != -1:379                    labels.append(label)380                    history.append(NEW_WHERE_OPS[label])381                if isinstance(s[0], dict):382                    stack.append(("root",s[0]))383                    # history.append("root")384                    dataset['root_tem_dataset'].append({385                        "question_tokens": question_tokens,386                        "ts": table_schema,387                        "history": history[:],388                        "gt_col": node[1][0][1][2],389                        "label": 0390                    })391                else:392                    dataset['root_tem_dataset'].append({393                        "question_tokens": question_tokens,394                        "ts": table_schema,395                        "history": history[:],396                        "gt_col": node[1][0][1][2],397                        "label": 1398                    })399                    # history.append("terminal")400            if len(labels) > 2:401                print(question_tokens)402            dataset['op_dataset'][-1]["label"] = labels403        elif node[0] == "where":404            history.append(node[0])405            hist, label = AndOrPredictor(question_tokens, node[1], table, history).generate_output()406            if label != -1:407                dataset['andor_dataset'].append({408                    "question_tokens": question_tokens,409                    "ts": table_schema,410                    "history": history[:],411                    "label":label412                })413            col_ret = ColPredictor(question_tokens, node[1], table, history, "where").generate_output()414            op_col_dict = dict()415            for h, l, s in col_ret:416                if l[0] == 0:417                    print("Warning: predicted 0 columns!")418                    continue419                dataset['col_dataset'].append({420                    "question_tokens": question_tokens,421                    "ts": table_schema,422                    "history": history[:],423                    "label": get_label_cols(with_join,fk_dict,l[1])424                })425                for col, sql_item in zip(l[1], s):426                    key = "{}{}{}".format(col[0][0], col[0][1], col[0][2])427                    if key not in op_col_dict:428                        op_col_dict[key] = [(sql_item, col[0])]429                    else:430                        op_col_dict[key].append((sql_item, col[0]))431                for key in op_col_dict:432                    stack.append(("col", "where", op_col_dict[key]))433        elif node[0] == "groupBy":434            history.append(node[0])435            col_ret = ColPredictor(question_tokens, node[1], table, history, node[0]).generate_output()436            agg_col_dict = dict()437            for h, l, s in col_ret:438                if l[0] == 0:439                    print("Warning: predicted 0 columns!")440                    continue441                dataset['col_dataset'].append({442                    "question_tokens": question_tokens,443                    "ts": table_schema,444                    "history": history[:],445                    "label": get_label_cols(with_join,fk_dict,l[1])446                })447                for col, sql_item in zip(l[1], s):448                    key = "{}{}{}".format(col[0][0], col[0][1], col[0][2])449                    if key not in agg_col_dict:450                        agg_col_dict[key] = [(sql_item, col[0])]451                    else:452                        agg_col_dict[key].append((sql_item, col[0]))453                for key in agg_col_dict:454                    stack.append(("col", node[0], agg_col_dict[key]))455def parser_item(question_tokens, sql, table, history, dataset):456    # try:457    #     question_tokens = item['question_toks']458    # except:459    #     print(item)460    # sql = item['sql']461    table_schema = [462        table["table_names"],463        table["column_names"],464        table["column_types"]465    ]466    history, label, sql = MultiSqlPredictor(question_tokens, sql, history).generate_output()467    dataset['multi_sql_dataset'].append({468        "question_tokens": question_tokens,469        "ts": table_schema,470        "history": history[:],471        "label": SQL_OPS[label]472    })473    history.append(label)474    history, label, sql = KeyWordPredictor(question_tokens, sql, history).generate_output()475    label_idxs = []476    for item in label[1]:477        if item in KW_DICT:478            label_idxs.append(KW_DICT[item])479    label_idxs.sort()480    dataset['keyword_dataset'].append({481        "question_tokens": question_tokens,482        "ts": table_schema,483        "history": history[:],484        "label": label_idxs485    })486    hist,label = AndOrPredictor(question_tokens,sql,table,history).generate_output()487    if label != -1:488        dataset['andor_dataset'].append({489            "question_tokens": question_tokens,490            "ts": table_schema,491            "history": hist[:]+["where"],492            "label": label493        })494    orderby_ret = DesAscPredictor(question_tokens, sql, table, history).generate_output()495    if orderby_ret:496        dataset['des_asc_dataset'].append({497            "question_tokens": question_tokens,498            "ts": table_schema,499            "history": orderby_ret[0][:],500            "label": orderby_ret[1]501        })502    col_ret = ColPredictor(question_tokens, sql, table, history).generate_output()503    agg_candidates = []504    op_candidates = []505    for h, l, s in col_ret:506        if l[0] == 0:507            print("Warning: predicted 0 columns!")508            continue509        dataset['col_dataset'].append({510            "question_tokens": question_tokens,511            "ts": table_schema,512            "history": h[:],513            "label": list(set([l[1][i][0][2] for i in range(min(len(l[1]),3))]))514        })515        for col, sql_item in zip(l[1], s):516            if h[-1] in ('where', 'having'):517                op_candidates.append((h + [col[0]], sql_item))518            if h[-1] in ('select', 'orderBy', 'having'):519                agg_candidates.append((h + [col[0]], sql_item))520            if h[-1] == "groupBy":521                label = 0522                if sql["having"]:523                    label = 1524                dataset['having_dataset'].append({525                    "question_tokens": question_tokens,526                    "ts": table_schema,527                    "history": h[:] + [col[0]],528                    "label": label529                })530    op_col_dict = dict()531    for h, sql_item in op_candidates:532        _, label, s = OpPredictor(question_tokens, sql_item, h).generate_output()533        if label == -1:534            continue535        key = "{}{}".format(h[-2], h[-1][2])536        label = NEW_WHERE_OPS[label]537        if key in op_col_dict:538            op_col_dict[key][1].append(label)539        else:540            op_col_dict[key] = [h[:], [label]]541        # dataset['op_dataset'].append({542        #     "question_tokens": question_tokens,543        #     "ts": table_schema,544        #     "history": h[:],545        #     "label": label546        # })547        if isinstance(s[0], dict):548            dataset['root_tem_dataset'].append({549                "question_tokens": question_tokens,550                "ts": table_schema,551                "history": h[:] + [label],552                "label": 0553            })554            parser_item(question_tokens, s[0], table, h[:] + [label], dataset)555        else:556            dataset['root_tem_dataset'].append({557                "question_tokens": question_tokens,558                "ts": table_schema,559                "history": h[:] + [label],560                "label": 1561            })562    for key in op_col_dict:563        # if len(op_col_dict[key][1]) > 1:564        #     print("same col has mult op ")565        dataset['op_dataset'].append({566            "question_tokens": question_tokens,567            "ts": table_schema,568            "history": op_col_dict[key][0],569            "label": op_col_dict[key][1]570        })571    agg_col_dict = dict()572    for h, sql_item in agg_candidates:573        _, label = AggPredictor(question_tokens, sql_item, h).generate_output()574        if label != 5:575            key = "{}{}".format(h[-2], h[-1][2])576            if key in agg_col_dict:577                agg_col_dict[key][1].append(label)578            else:579                agg_col_dict[key] = [h[:], [label]]580    for key in agg_col_dict:581        # if 5 in agg_col_dict[key][1]:582        #     print("none in agg label!!!")583        dataset['agg_dataset'].append({584            "question_tokens": question_tokens,585            "ts": table_schema,586            "history": agg_col_dict[key][0],587            "label": agg_col_dict[key][1]...

views.py

Source:views.py

...3from logger import Logger4from outputs.controllers import OutputController5from outputs.models import ShortUrl6from search.controllers import SearchController7def generate_output(request, url_identifier):8    Logger.Info('%s - generate_output - started' % __name__)9    Logger.Debug('%s - generate_output - started with url_identifier:%s' % (__name__, url_identifier))10    short_url = ShortUrl.Load(url_identifier)11    if not short_url:12        Logger.Error('%s - generate_output - error - url_identifier:%s not found' % (__name__, url_identifier))13        Logger.Info('%s - generate_output - finished' % __name__)14        raise Http40415    dashboard_id = short_url.dashboard_id16    collection_id = short_url.collection_id17    output_id = short_url.output_id18    dashboard = DashboardsController.GetDashboardById(dashboard_id)19    if not dashboard:20        Logger.Error('%s - generate_output - error - dashboard_id:%s not found' % (__name__, dashboard_id))21        Logger.Info('%s - generate_output - finished' % __name__)22        raise Http40423    collection = None24    for c in dashboard.collections:25        if c['id'] == collection_id:26            collection = c27            break28    if not collection:29        Logger.Error('%s - generate_output - error - collection_id:%s not found' % (__name__, collection_id))30        Logger.Info('%s - generate_output - finished' % __name__)31        raise Http40432    output = None33    if 'outputs' in collection:34        for o in collection['outputs']:35            if o['id'] == output_id:36                output = o37                break38    if not output:39        Logger.Error('%s - generate_output - error - output_id:%s not found' % (__name__, output_id))40        Logger.Info('%s - generate_output - finished' % __name__)41        raise Http40442    search_query_replacements = []43    limit = request.GET.get('limit')44    if limit:45        try:46            limit = int(limit)47            search_query_replacements.append({48                'pattern':r'rows=[0-9]+',49                'replacement':'rows=%i' % limit50            })51        except ValueError:52            pass53    sc = SearchController(collection)54    search_results = sc.run_search_and_return_results(search_query_replacements=search_query_replacements)55    oc = OutputController(output)56    content, content_type = oc.generate_output(search_results)57    Logger.Info('%s - generate_output - finished' % __name__)58    response = HttpResponse(content=content, content_type=content_type)59    if 'csv' in content_type:60        response['Content-Disposition'] = 'attachment; filename=metalayer_output.csv'61    if 'ms-excel' in content_type:62        response['Content-Disposition'] = 'attachment; filename=metalayer_output.xlsx'63    response['Access-Control-Allow-Origin'] = '*'...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.