KaneAI - World’s First E2E Software Testing Agent.Book a Demo Join TestMu Conference

How to use is_not_nan method in assertpy

Best Python code snippet using assertpy_python

lsst_single_exp_cat_match_aper_corr.py

Source:lsst_single_exp_cat_match_aper_corr.py

1from astropy.io import ascii2import numpy as np3import matplotlib4matplotlib.use("TkAgg")5import matplotlib.pyplot as plt6import matplotlib.colors7from astropy.coordinates import SkyCoord8from astropy import coordinates as coords9import astropy.units as u10from astropy.io import fits11from astropy import wcs12from astropy.table import Table, vstack13sex_dir=("/Users/duhokim/work/abell/sex/cat/")14plot_dir=("/Users/duhokim/work/abell/plot/")15cat_dir=("/Users/duhokim/work/abell/cat/")16work_dir=("/Users/duhokim/work/abell/")17# lsst_dir=("/Users/duhokim/lsst_stack/demo_data_newinstall_decam/DATA/rerun/coaddForcedPhot/deepCoadd-results/")18# sav_dir=("/Users/duhokim/lsst_stack/demo_data_newinstall_decam/")19lsst_dir=("/Users/duhokim/lsst_stack/demo_data_orig_20_intersect/DATA/rerun/coaddForcedPhot/deepCoadd-results/")20sav_dir=("/Users/duhokim/lsst_stack/demo_data_orig_20_intersect/")21visit_nums = ['0350110', '0350835']22'''23patches = [24    ['0,2', '0,3', '0,4', '1,3', '1,4', '1,5',        '2,2',               '3,1', '3,3',        '4,1', '4,2', '4,3',25     '4,5', '5,1', '5,2', '5,3', '5,5', '6,2', '6,3'],26    ['0,2', '0,3', '0,4', '1,3', '1,4', '1,5', '2,1', '2,2', '2,6', '3,0', '3,1', '3,3', '4,0', '4,1', '4,2', '4,3',27     '4,5',        '5,2', '5,3', '5,5', '6,2', '6,3']28    ]29patches_all = ['0,2', '0,3', '0,4', '1,3', '1,4', '1,5', '2,2', '3,1', '3,3', '4,1', '4,2', '4,3', '4,5', '5,2', '5,3',30               '5,5', '6,2', '6,3']31'''32patches = [33    ['0,2', '0,3',        '1,2',       '2,2', '2,3',              '3,2',               '4,2',        '5,2', '6,2'],34    ['0,2', '0,3', '1,1', '1,2', '2,0',             '3,0', '3,1', '3,2', '4,0', '4,1', '4,2', '5,1', '5,2']35    ]36patches_all = ['0,2', '0,3', '1,2', '3,2', '4,2', '5,2']37ver = 'v1.1'38sdss_mag_sys = 'petroMag_'39sex_mag_sys = 'MAG_PETRO'40class_star_lim = 0.941mag_lim = [99, 99, 99]    # limit magnitude for analysis42mag_ran = [25, 13]43clusters = ['A2670']44bands = ['g', 'r']45fn =  ['60_0420', '300_0351']46single_dqm_fn = ['A2670_gd_60_0420_21aug.fits', 'A2670_rd_300_0351_19aug.fits']47exp = [60, 300]48seeing = ['1.28', '0.92']49X = [1.26, 1.41]50a = [0.179, 0.567]51b = [-0.036, -0.136]52# gal_ext = [ [0.159, 0.124, 0.086], # irsa.ipac.caltech.edu, S and F (2011)53#             [0.188, 0.146, 0.101],54#             [0.157, 0.122, 0.085]]55gal_ext = [0, 0]56max_sep = 1.057ccd_x = 204658ccd_y = 409459fig, axs = plt.subplots(2, 2, tight_layout = True, figsize = (10, 10))60sex_cat = ascii.read(sex_dir + 'DECam_19_21_aug_2014_single_best_exposure_SEx_cat_A2670_match_rad_1as_'61                                                                  'Gal_ext_corrected_' + ver + '.txt')62sex_coords = SkyCoord(sex_cat['ALPHA_J2000'], sex_cat['DELTA_J2000'], unit='deg')63for j in range(0, len(bands)):64    for i in range(0, len(patches[j])):65        rMags = np.load(sav_dir+bands[j]+'mag'+patches[j][i][0]+patches[j][i][2]+'.npy')66        lsst_all = Table.read(lsst_dir + bands[j] + '/0/' + patches[j][i] + '/forcedSrc-'+bands[j]+'-0-'+67                              patches[j][i]+'.fits')68        is_not_nan = [(not np.isnan(rMags[x][0])) & (not np.isnan(rMags[x][1])) & (rMags[x][1] < 0.1) for x in range(0, len(rMags))]69        rMags = rMags[is_not_nan]70        lsst_all = lsst_all[is_not_nan]71        lsst_coords = SkyCoord(lsst_all['coord_ra'], lsst_all['coord_dec'], unit=(u.rad, u.rad))72        idx, d2d, d3d = sex_coords.match_to_catalog_sky(lsst_coords)73        sep_constraint = d2d.arcsec < max_sep74        sex_matches = sex_cat[sep_constraint]75        lsst_matches = rMags[idx[sep_constraint]]76        axs[0, j].scatter(sex_matches['MAG_AUTO_'+bands[j]],77                          lsst_matches[:, 0],78                          alpha=0.2, s=1)79        axs[0, j].scatter(sex_matches['MAG_AUTO_'+bands[j]],80                          lsst_matches[:, 0],81                          alpha=0.2, s=1)82for i in range(0, len(patches_all)):83    gMags = np.load(sav_dir+'gmag'+patches_all[i][0]+patches_all[i][2]+'.npy')84    rMags = np.load(sav_dir+'rmag'+patches_all[i][0]+patches_all[i][2]+'.npy')85    is_not_nan = [(not np.isnan(gMags[x][0])) & (not np.isnan(gMags[x][1])) & (not np.isnan(rMags[x][0])) &86                  (not np.isnan(rMags[x][1])) & (gMags[x][1] < 0.1) & (rMags[x][1] < 0.1) for x in range(0, len(rMags))]87    if i == 0:88        lsst_g = gMags[is_not_nan, 0]89        lsst_r = rMags[is_not_nan, 0]90    else:91        lsst_g = np.append(lsst_g, gMags[is_not_nan, 0], axis=0)92        lsst_r = np.append(lsst_r, rMags[is_not_nan, 0], axis=0)93axs[1, 0].scatter(sex_cat['MAG_AUTO_'+bands[1]],94                  sex_cat['MAG_AUTO_'+bands[0]] - sex_cat['MAG_AUTO_'+bands[1]],95                  alpha=0.2, s=1, label='SEx (CLASS_STAR < 0.9 & DQM_cen = 0,128)')96axs[1, 1].scatter(lsst_r,97                  lsst_g - lsst_r,98                  alpha=0.2, s=1, label='LSST (merr < 0.1)')99axs[0, 0].plot(mag_ran, mag_ran, linestyle=':', alpha=0.5)100axs[0, 1].plot(mag_ran, mag_ran, linestyle=':', alpha=0.5)101axs[0, 0].text(27.5, 13, 'A2670', fontsize=24)102axs[0, 0].text(27.5, 13, 'A2670', fontsize=24)103axs[0, 0].set_title('g band (60s exposure)')104axs[0, 1].set_title('r band (300s exposure)')105axs[0, 0].set_xlabel('SEx (MAG_AUTO)')106axs[0, 1].set_xlabel('SEx (MAG_AUTO)')107axs[0, 0].set_ylabel('LSST (base_PsfFlux) (merr < 0.1)')108axs[0, 1].set_ylabel('LSST (base_PsfFlux) (merr < 0.1)')109axs[0, 0].set_xlim(mag_ran)110axs[0, 1].set_xlim(mag_ran)111axs[0, 0].set_ylim(mag_ran)112axs[0, 1].set_ylim(mag_ran)113axs[1, 0].set_xlabel('r')114axs[1, 0].set_ylabel('g-r')115axs[1, 0].set_xlim(mag_ran)116axs[1, 0].set_ylim([-3, 5])117axs[1, 0].legend()118axs[1, 1].set_xlabel('r')119axs[1, 1].set_ylabel('g-r')120axs[1, 1].set_xlim(mag_ran)121axs[1, 1].set_ylim([-3, 5])122axs[1, 1].legend()...

lidar2d.py

Source:lidar2d.py

1import numpy as np2from math import cos, sin3class LiDAR2D:4    def __init__(self, robot_ptr):5        # specification: default values from SICK TiM571 datasheet6        self.range_min = 0.05   # meter7        self.range_max = 8      # meter (actually this is the range typical of the sensor)8        # self.range_max = 25   #9        self.fov = 360          # in degree10        self.resolution = 2     # per degree11        self.angle_min_radian = lambda: - self.fov / 2 * np.pi / 180.12        self.angle_max_radian = lambda: + self.fov / 2 * np.pi / 180.13        self.angle_increment_radian = lambda: (1 / self.resolution) * np.pi / 18014        self.num_pts = lambda: self.fov * self.resolution15        self.scanning_freq = 15.  # Hz16        self.time_increment = lambda: 1. / self.scanning_freq17        self.systematic_error_abs = 0.060   # m18        self.statistical_error = 0.020      # m19        self.data_type = np.float3220        self.robot_ptr = robot_ptr21        # scan results22        class Data:23            def __init__(self):24                self.last_rotated_rays = []25                self.last_points = []26                self.last_range_data = []27                self.last_intensities = []28                self.last_occupancy_gridmap = []29                self.occlusion_hist = dict((ii, []) for ii in range(10))30        self.data = Data()31    def scan(self, world):32        angles = np.deg2rad(np.arange(-self.fov / 2, self.fov / 2, 1 / self.resolution))33        rays_end = (np.array([np.cos(angles), np.sin(angles)]) * self.range_max).T34        rays_origin = np.zeros_like(rays_end)35        rays = np.stack([rays_origin, rays_end], axis=1)36        self.data.last_rotated_rays = rays37        # Rotate (to robot face)38        rot_matrix = np.array([[cos(self.robot_ptr.orien), -sin(self.robot_ptr.orien)],39                               [sin(self.robot_ptr.orien), cos(self.robot_ptr.orien)]])40        self.data.last_rotated_rays[:, 1, :] = np.matmul(rot_matrix, self.data.last_rotated_rays[:, 1, :].transpose()).transpose()41        # Translate (to robot position)42        self.data.last_rotated_rays[:, 0, :] = self.data.last_rotated_rays[:, 0, :] + self.robot_ptr.pos43        self.data.last_rotated_rays[:, 1, :] = self.data.last_rotated_rays[:, 1, :] + self.robot_ptr.pos44        # scan the obstacles45        all_intersects = [self.data.last_rotated_rays[:, 1]]46        for obs in world.obstacles:47            results, intersect_pts_ = obs.intersect_many(self.data.last_rotated_rays)48            is_not_nan = 1 - np.any(np.isnan(intersect_pts_), axis=1)49            results = np.bitwise_and(results, is_not_nan)50            intersect_pts = np.stack([res * intersect_pts_[ii] + (1-res) * (self.range_max/np.sqrt(2) + self.robot_ptr.pos)51                                      for ii, res in enumerate(results)])52            all_intersects.append(intersect_pts)53        # scan the pedestrians54        for kk, ped in enumerate(world.crowds):55            dist = np.linalg.norm(ped.pos - self.robot_ptr.pos) - ped.radius56            # ignore the objects that are farther than max_range of the lidar from the robot57            if dist > self.range_max:58                continue59            ped_geometry = ped.geometry()60            results, intersect_pts_ = ped_geometry.intersect_many(self.data.last_rotated_rays)61            is_not_nan = 1 - np.any(np.isnan(intersect_pts_), axis=1)62            results = np.bitwise_and(results, is_not_nan)63            intersect_pts = np.stack([res * intersect_pts_[ii] + (1-res) * (self.range_max/np.sqrt(2) + self.robot_ptr.pos)64                                      for ii, res in enumerate(results)])65            all_intersects.append(intersect_pts)66        # combine the intersection points67        all_intersects = np.stack(all_intersects)68        dists = all_intersects - self.data.last_rotated_rays[0, 0]69        dists = np.linalg.norm(dists, axis=2)70        min_dist_idx = np.argmin(dists, axis=0)71        self.data.last_points = np.stack([all_intersects[ind, ii] for ii, ind in enumerate(min_dist_idx)])72        # Todo: add gaussian noise73        # which agents are occluded74        min_dist = np.min(dists, axis=0)75        for ii in range(len(world.obstacles), len(dists)):76            rays_might_have_hit_i = dists[ii] < (self.range_max - 1E-3)77            rays_have_hit_i = np.bitwise_and((dists[ii] - 1E-3) < min_dist, rays_might_have_hit_i)78            if np.sum(rays_might_have_hit_i) != 0:79                occ_i = 1 - np.sum(rays_have_hit_i) / (np.sum(rays_might_have_hit_i) + 1E-9)80                bin_idx = int(min(dists[ii]))81            else:  # put it in the bin, out of lidar range (>8.0m)82                occ_i = 183                bin_idx = int(round(self.range_max))84            self.data.occlusion_hist[bin_idx].append(int(round(occ_i * 100)))85        self.data.last_range_data = np.sqrt(np.power(self.data.last_points[:, 0] - self.robot_ptr.pos[0], 2) +86                                            np.power(self.data.last_points[:, 1] - self.robot_ptr.pos[1], 2))87        # Todo: set lidar intensities...

outlier_detection.py

Source:outlier_detection.py

1import numpy as np2import fbprophet as fbp3import statsmodels.api as sm4def find_outliers_prophet(data):5    """Iteratively find outliers, excluding new outliers and refitting.6    Outlier: point lying outside interval_width of fitted model.7    :param data: Data.8    """9    df = data.get_df()10    outlier = np.zeros(len(df), dtype=bool)11    for i in range(data.n_iterations):12        m = fbp.Prophet(**{"interval_width": 0.99, **data.method_kws})13        m.fit(df)14        df_pred = m.predict(df)15        new_outlier = ((df_pred.yhat_lower > df.y) | (df_pred.yhat_upper < df.y)).values16        if new_outlier.sum() == 0:17            break18        df.loc[outlier, "y"] = np.nan19        outlier |= new_outlier20    df_pred["outlier"] = outlier21    return m, df_pred22def find_outliers_lowess(data):23    """Iteratively find outliers, excluding new outliers and refitting.24    Outlier: Point which has large residual compared to nearby (=rolling window) points.25    :param data: Data.26    """27    frac = data.method_kws.get("frac", 1 / 5)28    outlier_n_std = data.method_kws.get("outlier_n_std", 2.5)29    rolling_window_size = data.method_kws.get("rolling_window_size")30    rolling_window_min_periods = data.method_kws.get("rolling_window_min_periods", 4)31    # Minimum requirements for window32    rolling_window_size = rolling_window_size or max(int(frac * len(data.y)), 2)33    rolling_window_min_periods = min(rolling_window_min_periods, rolling_window_size)34    df = data.get_df()35    df["y_orig"] = df.y36    outlier = np.zeros(len(df), dtype=bool)37    for i in range(data.n_iterations):38        y_lowess = sm.nonparametric.lowess(df.y, df.ds, frac=frac, return_sorted=False)39        df["res"] = (df.y_orig - y_lowess).abs()40        # Use rolling window to allow for changes in noise level over "time"41        rolling = df.res.rolling(rolling_window_size, min_periods=rolling_window_min_periods, center=True)42        df["res_ma"] = rolling.mean().bfill().ffill()43        df["res_mstd"] = rolling.std().bfill().ffill()44        # Classify datapoints far from rolling mean as outliers45        new_outlier = (df.res - df.res_ma).abs() > outlier_n_std * df.res_mstd46        if new_outlier.sum() == 0:47            break48        outlier |= new_outlier49        df["y"] = df.y_orig50        df.loc[outlier, "y"] = np.nan51    df["yhat"] = y_lowess52    df["yhat_upper"] = df.yhat + outlier_n_std * np.abs(df.res_mstd)53    df["yhat_lower"] = (df.yhat - outlier_n_std * np.abs(df.res_mstd))54    # Interpolation for outliers55    is_nan, is_not_nan = np.isnan(y_lowess), ~np.isnan(y_lowess)56    x = np.arange(len(df))57    for col in ("yhat", "yhat_upper", "yhat_lower"):58        df[col] = np.interp(x, x[is_not_nan], df[col][is_not_nan])59    # Cleanup60    df.drop(columns=["y_orig", "res", "res_ma", "res_mstd"], inplace=True)61    df["outlier"] = outlier62    return None, df63def find_outliers(data):64    if data.method == "prophet":65        m, df_pred = find_outliers_prophet(data)66    elif data.method == "lowess":67        m, df_pred = find_outliers_lowess(data)68    else:69        raise ValueError("Invalid method (%s)!" % data.method)...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.