KaneAI - World’s First E2E Software Testing Agent.Book a Demo Join TestMu Conference

How to use filtered_models method in dbt-osmosis

Best Python code snippet using dbt-osmosis_python

process_dataset.py

Source:process_dataset.py

1import argparse2import logging3import pandas as pd4import numpy as np5from sklearn.metrics import average_precision_score6import utils7import models8import soluto_log9def parse_args():10    parser = argparse.ArgumentParser(description='predictive model testing for photography segment')11    parser.add_argument('data_path', help='path to directory with the raw data')12    requiredNamed = parser.add_argument_group('required named arguments')13    requiredNamed.add_argument('--folds',  required=True, type=int,14                               help='number of folds in K-fold cross-validation')15    requiredNamed.add_argument('--test_size', type=float,16                               help='portion of test set out of all records')17    return parser.parse_args()18def models_variations(X_train, X_test, y_train, y_test, folds):19    logger = logging.getLogger('log')20    # NOTE: X sets might contain raws with only zeros.21    apps = utils.get_apps(X_train.columns)22    google_apps = [i for i in apps if 'google.android' in i]23    logger.debug('apps with "google.android" in package name:\n{}'.format(google_apps))24    carrier_apps_pattern = set([i for i in X_train['carrier_name'].unique()])25    carrier_apps = [i for i in apps if (carrier_apps_pattern & set(i.split('.')))]26    logger.debug('apps with a carrier name in package name:\n{}'.format(carrier_apps))27    # .sec. is for samsung apps.28    vendor_apps_pattern = set([i for i in X_train['device_vendor'].unique()] + ['sec'])29    vendor_apps = [i for i in apps if (vendor_apps_pattern & set(i.split('.')))]30    logger.debug('apps with a vendor name in package name:\n{}'.format(vendor_apps))31    pre_installed = {}32    for property in ['carrier_name', 'device_model', 'device_vendor']:33        pre_installed_apps = \34            utils.get_top_by_property(X_train[[property] + apps], property, 100, 0.9)35        pre_installed[property] = pre_installed_apps36        logger.debug('{property} pre installed apps:\n{apps}'.format(37            property=property, apps=pre_installed_apps.to_string()))38    # hot-one encode device model39    # NOTE: X_train and X_test might have inconsistent columns due to rare device models40    device_models = X_train['device_model']41    X_train = pd.get_dummies(X_train, columns=['device_model'], prefix='', prefix_sep='')42    X_train = pd.concat([X_train, device_models], axis=1)43    logger.info('train shape: {}'.format(X_train.shape))44    device_models_test = X_test['device_model']45    X_test = pd.get_dummies(X_test, columns=['device_model'], prefix='', prefix_sep='')46    X_test = pd.concat([X_test, device_models_test], axis=1)47    device_models = X_train['device_model'].unique()48    logger.info('all apps ; devices models with variance > 0.0099')49    filtered_models = utils.filter_low_var_columns(X_train[device_models], 0.0099)50    features = np.append(filtered_models, apps)51    model_df = X_train[features].copy()52    model = models.logistic_regression(model_df, y_train, folds)53    logger.info('test average precision: {}'.format(54        average_precision_score(y_test,55                                model.predict_proba(X_test[model_df.columns])[:, 1])))56    logger.info('apps with variance > 0.09 ; devices models with variance > 0.0099')57    filtered_apps = utils.filter_low_var_columns(X_train[apps], 0.09)58    filtered_models = utils.filter_low_var_columns(X_train[device_models], 0.0099)59    features = np.append(filtered_models, filtered_apps)60    model_df = X_train[features].copy()61    model = models.logistic_regression(model_df, y_train, folds)62    logger.info('test average precision: {}'.format(63        average_precision_score(y_test,64                                model.predict_proba(X_test[model_df.columns])[:, 1])))65    logger.info('apps with variance > 0.09 ; devices models with variance > 0.0099 ; clean rows with 0 apps')66    filtered_apps = utils.filter_low_var_columns(X_train[apps], 0.09)67    filtered_models = utils.filter_low_var_columns(X_train[device_models], 0.0099)68    features = np.append(filtered_models, filtered_apps)69    model_df = X_train[features].copy()70    model_df = model_df[model_df.sum(axis=1) > 0]71    model = models.logistic_regression(model_df, y_train, folds)72    logger.info('test average precision: {}'.format(73        average_precision_score(y_test,74                                model.predict_proba(X_test[model_df.columns])[:, 1])))75    logger.info('apps with variance > 0.09 + remove apps by name ; devices models with variance > 0.0099')76    filtered_apps = [i for i in apps if i not in google_apps + carrier_apps + vendor_apps]77    filtered_apps = utils.filter_low_var_columns(X_train[filtered_apps], 0.09)78    filtered_models = utils.filter_low_var_columns(X_train[device_models], 0.0099)79    features = np.append(filtered_models, filtered_apps)80    model_df = X_train[features].copy()81    model = models.logistic_regression(model_df, y_train, folds)82    logger.info('test average precision: {}'.format(83        average_precision_score(y_test,84                                model.predict_proba(X_test[model_df.columns])[:, 1])))85    logger.info('apps with variance > 0.09 + remove apps by name ; devices models with variance > 0.0099 ; clean rows with 0 apps')86    filtered_apps = [i for i in apps if i not in google_apps + carrier_apps + vendor_apps]87    filtered_apps = utils.filter_low_var_columns(X_train[filtered_apps], 0.09)88    filtered_models = utils.filter_low_var_columns(X_train[device_models], 0.0099)89    features = np.append(filtered_models, filtered_apps)90    model_df = X_train[features].copy()91    model_df = model_df[model_df.sum(axis=1) > 0]92    model = models.logistic_regression(model_df, y_train, folds)93    logger.info('test average precision: {}'.format(94        average_precision_score(y_test,95                                model.predict_proba(X_test[model_df.columns])[:, 1])))96    logger.info('remove apps by pre installed ; devices models with variance > 0.0099')97    model_df = X_train.copy()98    model_df_test = X_test.copy()99    for property in ['carrier_name', 'device_model', 'device_vendor']:100        for p, apps_replace in pre_installed[property].groupby(level=0):101            apps_replace_list = apps_replace.index.get_level_values('level_1').tolist()102            model_df.loc[model_df[property] == p, apps_replace_list] = 0103            model_df_test.loc[model_df_test[property] == p, apps_replace_list] = 0104    filtered_models = utils.filter_low_var_columns(model_df[device_models], 0.0099)105    features = np.append(filtered_models, apps)106    model_df = model_df[features]107    model = models.logistic_regression(model_df, y_train, folds)108    logger.info('test average precision: {}'.format(109        average_precision_score(y_test,110                                model.predict_proba(model_df_test[model_df.columns])[:, 1])))111    logger.info('apps with variance > 0.09 + remove apps by pre installed ; devices models with variance > 0.0099')112    model_df = X_train.copy()113    model_df_test = X_test.copy()114    for property in ['carrier_name', 'device_model', 'device_vendor']:115        for p, apps_replace in pre_installed[property].groupby(level=0):116            apps_replace_list = apps_replace.index.get_level_values('level_1').tolist()117            model_df.loc[model_df[property] == p, apps_replace_list] = 0118            model_df_test.loc[model_df_test[property] == p, apps_replace_list] = 0119    filtered_apps = utils.filter_low_var_columns(model_df[apps], 0.09)120    filtered_models = utils.filter_low_var_columns(model_df[device_models], 0.0099)121    features = np.append(filtered_models, filtered_apps)122    model_df = model_df[features]123    model = models.logistic_regression(model_df, y_train, folds)124    logger.info('test average precision: {}'.format(125        average_precision_score(y_test,126                                model.predict_proba(model_df_test[model_df.columns])[:, 1])))127    logger.info('apps with variance > 0.09')128    filtered_apps = utils.filter_low_var_columns(X_train[apps], 0.09)129    features = filtered_apps130    model_df = X_train[features].copy()131    model = models.logistic_regression(model_df, y_train, folds)132    logger.info('test average precision: {}'.format(133        average_precision_score(y_test,134                                model.predict_proba(X_test[model_df.columns])[:, 1])))135    logger.info('apps with variance > 0.09 + remove apps by pre installed')136    model_df = X_train.copy()137    model_df_test = X_test.copy()138    for property in ['carrier_name', 'device_model', 'device_vendor']:139        for p, apps_replace in pre_installed[property].groupby(level=0):140            apps_replace_list = apps_replace.index.get_level_values('level_1').tolist()141            model_df.loc[model_df[property] == p, apps_replace_list] = 0142            model_df_test.loc[model_df_test[property] == p, apps_replace_list] = 0143    filtered_apps = utils.filter_low_var_columns(model_df[apps], 0.09)144    features = filtered_apps145    model_df = model_df[features]146    model = models.logistic_regression(model_df, y_train, folds)147    logger.info('test average precision: {}'.format(148        average_precision_score(y_test,149                                model.predict_proba(model_df_test[model_df.columns])[:, 1])))150def main():151    args = parse_args()152    logger = soluto_log.get_logger(filename='photography_model')153    try:154        logger.info('start')155        X_train, X_test, y_train, y_test = utils.load_ready_train_test(args.data_path)156        models_variations(X_train, X_test, y_train, y_test, args.folds)157    except Exception as e:158        logger.exception('Exception')159    finally:160        logger.info('end')161if __name__ == '__main__':...

play.py

Source:play.py

1from sources import play_agent, start_carla, get_hparams, ConsoleStats, CarlaEnvSettings2import os3from threading import Thread4# Replacement for shared multiprocessing object when playing5class PauseObject:6    value = 07if __name__ == '__main__':8    9    # Load hparams if they are being saved by trainer10    hparams = get_hparams(playing=True)11    print('Saved models:')12    # Check paths from hparams, filter out models whose files exist on disk13    filtered_models = {}14    for model in hparams['models']:15        if os.path.isfile(model):16            filtered_models[int(model.split('.')[0].split('_')[-1])] = model17    filtered_models = list(dict(sorted(filtered_models.items(), key=lambda kv: kv[0])).values())18    for index, model in enumerate(filtered_models):19        print(f'{index+1}. {model}')20    # Ask user for a model to choose21    model = filtered_models[int(input('Choose the model (empty - most recent one): ') or '0') - 1]22    print('Starting...')23    # Kill Carla processes if there are any and start simulator24    start_carla(playing=True)25    # Run Carla settings (weather, NPC control) in a separate thread26    pause_object = PauseObject()27    carla_settings = CarlaEnvSettings(0, [pause_object], car_npcs=hparams['car_npcs'])28    carla_settings_thread = Thread(target=carla_settings.update_settings_in_loop, daemon=True)29    carla_settings_thread.start()30    # Play31    print(f'Starting agent ({model})...')...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.