KaneAI - World’s First E2E Software Testing Agent.Signup for Private Beta Join TestMu Conference

How to use clearCustomQueryHandlers method in Puppeteer

Best JavaScript code snippet using puppeteer

scraper.js

Source:scraper.js

1console.log("----------------- EXECUTING FILE: scraper.js -----------------")2const puppeteer = require("puppeteer-extra");3const { Cluster } = require('puppeteer-cluster');4const fs = require('fs-extra');5const axios = require("axios");6const links = {7    "Challenger": [8        {9            "requests": [10                {11                    "indexName": "hachisearchengine",12                    "params": "filters=active_sites%3AHSG&maxValuesPerFacet=1&query=&hitsPerPage=1000&highlightPreTag=__ais-highlight__&highlightPostTag=__%2Fais-highlight__&page=0&tagFilters=&facetFilters=%5B%5B%22boutiquecates.subcategory%3APc%20%26%20Notebooks%20%3E%20Notebooks%20%26%20Desktops%20%3E%20Monitors%22%5D%5D"13                }, // For normal monitors14                {15                    "indexName": "hachisearchengine",16                    "params": "filters=active_sites%3AHSG&maxValuesPerFacet=1&query=&hitsPerPage=1000&highlightPreTag=__ais-highlight__&highlightPostTag=__%2Fais-highlight__&page=0&tagFilters=&facetFilters=%5B%5B%22boutiquecates.subcategory%3AGaming%20%3E%20Pc%20Gaming%20%3E%20Gaming%20Monitor%22%5D%5D"17                }, // for Gaming monitors18            ]19        }20    ],21    "Best Denki": ['https://www.bestdenki.com.sg/it-mobile/computer/monitors.html'],22    "Courts": ['https://www.courts.com.sg/computing-mobile/monitors-projectors/monitors?product_list_limit=32'],23    "Harvey Norman": ['https://www.harveynorman.com.sg/computing/computers-en/monitors-en/'],24    "Gain City": ['https://www.gaincity.com/catalog/category/166/monitors'],25    "ACER Store": ['https://store.acer.com/en-sg/monitors?product_list_limit=all'],26    "DELL Store": ['https://www.dell.com/en-sg/shop/monitors/ar/6481'],27    "HP Store": ['https://www.hp.com/sg-en/shop/monitors.html?product_list_limit=30'],28    "LENOVO Store": ['https://www.lenovo.com/sg/en/accessories-and-monitors/dc/monitors?sort=sortBy&currentResultsLayoutType=grid&visibleDatas=facet_Group%3AGroup%3AMonitors'],29    "LG Store": ['https://www.lg.com/sg/consumer-monitors/view-all']30};31const headless = true32const StealthPlugin = require('puppeteer-extra-plugin-stealth');33// const { clearCustomQueryHandlers } = require("puppeteer");34puppeteer.use(StealthPlugin());35const pageTimeout = 5 * 1000 * 6036const CLUSTEROPTS = {37    puppeteer,38    concurrency: Cluster.CONCURRENCY_PAGE,39    maxConcurrency: 2,40    timeout: 30 * 1000 * 60,41    sameDomainDelay: 500,42    workerCreationDelay: 500,43    monitor: true,44    puppeteerOptions: {45        headless,46        // TOGGLE ARGS FOR PUSH47        args: [48            // '--disable-setuid-sandbox',49            // '--disable-dev-shm-usage',50            // '--disable-accelerated-2d-canvas',51            // '--no-first-run',52            // '--no-zygote',53            // '--single-process', // <- this one doesn't works in Windows54            '--disable-gpu',55            // '--no-sandbox'56        ]57    }58}59const brands = []60    ; (async () => {61        const delay = ms => new Promise(resolve => setTimeout(resolve, ms))62        try {63            const browser = await puppeteer.launch({ headless, args: ['--no-sandbox'] });64            const dictionary = await fs.readJSON(`${process.cwd()}/data/dictionary.json`)65            const extras = await fs.readJSON(`${process.cwd()}/data/extras.json`)66            await fs.ensureDir(`${process.cwd()}/data/raw/monitors`)67            await fs.ensureFile(`${process.cwd()}/data/raw/monitors/harvey.json`)68            await fs.ensureFile(`${process.cwd()}/data/raw/monitors/challenger.json`)69            await fs.ensureFile(`${process.cwd()}/data/raw/monitors/gain.json`)70            await fs.ensureFile(`${process.cwd()}/data/raw/monitors/courts.json`)71            await fs.ensureFile(`${process.cwd()}/data/raw/monitors/best.json`)72            // await fs.ensureFile(`${process.cwd()}/data/raw/monitors/lg.json`) 73            await fs.ensureFile(`${process.cwd()}/data/raw/monitors/acer.json`)74            // await fs.ensureFile(`${process.cwd()}/data/raw/monitors/asus.json`) // No monitors75            await fs.ensureFile(`${process.cwd()}/data/raw/monitors/dell.json`)76            await fs.ensureFile(`${process.cwd()}/data/raw/monitors/hp.json`)77            await fs.ensureFile(`${process.cwd()}/data/raw/monitors/lenovo.json`)78            await gain()79            await acer()80            await dell() 81            await hp()82            await lenovo()83            await harvey()84            await challenger()85            await courts() // Illegal86            await best()87            /* SECTION FOR ACER */88            async function acer() {89                console.log("PUPPETEER: Scraping Acer Store")90                const acerPage = await browser.newPage()91                await acerPage.goto(links["ACER Store"][0], { waitUntil: 'networkidle0' })92                await acerPage.exposeFunction("cleaner", cleaner)93                let acerData = await acerPage.evaluate(async () => {94                    try {95                        let items = document.querySelectorAll(".product-item")96                        let products = []97                        for (let i = 0; i < items.length; i++) {98                            let item = items[i]99                            let childName = item.querySelector(".product-item-link").innerText100                            let price = item.querySelector('.price').innerText101                            let brand = "ACER"102                            let link = item.querySelector(".product-item-link").getAttribute("href")103                            let model_ID = await cleaner(childName.includes("|") ? childName.split("|")[1].trim() : childName.trim())104                            let image_url = item.querySelector(".product-image-photo").src105                            let instock = item.querySelector(".stock.available") ? "" : "c-false"106                            products.push({107                                name: childName, price, brand, link, model_ID, image_url, instock,108                                location: "ACER Store"109                            })110                        }111                        return products112                    } catch (e) {113                        console.log(e)114                        return e115                    }116                })117                await acerPage.close()118                // Write to file119                fs.writeFile(`${process.cwd()}/data/raw/monitors/acer.json`, JSON.stringify(acerData), (err, file) => {120                    if (err) console.log(err)121                })122                console.log("PUPPETEER: Completed scraping Acer Store")123                return true124            }125            /* SECTION FOR DELL Store (inoperable) */126            async function dell() {127                try {128                    console.log("PUPPETEER: Scraping DELL Store")129                    const dellPage = await browser.newPage()130                    await dellPage.goto(links['DELL Store'][0], { waitUntil: 'domcontentloaded' })131                    let pages = await dellPage.$eval(("#middle-content > div > div.accessories-result-product-stack > div.pageinfo-control.hide-xs.hide-sm.hide-md > div:nth-child(1) > div > span.resultcount"), e => Math.ceil(Number(e.innerText) / 12))132                    console.log("Found " + pages + " for Dell")133                    await dellPage.close()134                    const cluster = await Cluster.launch(CLUSTEROPTS);135                    for (let i = 1; i <= pages; i++) {136                        let url = `${links['DELL Store'][0]}?page=${i}`137                        console.log(url)138                        cluster.queue({139                            url, i140                        })141                    }142                    const DELLProducts = []143                    await cluster.task(async ({ page, data }) => {144                        let url = data.url145                        let i = data.i146                        await page.exposeFunction('cleaner', cleaner)147                        console.log("Scraping page " + i + " of " + pages)148                        await page.goto(url, { waitUntil: 'networkidle0', timeout: pageTimeout })149                        let products = await page.evaluate(async () => {150                            try {151                                let items = document.querySelectorAll(".stack-accessories.ps-stack")152                                let products = []153                                for (let i = 0; i < items.length; i++) {154                                    let item = items[i]155                                    let childName = item.querySelector(".ps-title > a").innerText156                                    let price = item.querySelector('.smart-popover-btn').innerText157                                    let brand = "DELL"158                                    let link = item.querySelector(".ps-title > a").getAttribute("href")159                                    if (!link.match(/^https?:\/\//g)) link = "https:" + link160                                    let model_ID = await cleaner(childName)161                                    // let model_ID = await cleaner(childName.includes("|") ? childName.split("|")[1].trim() : childName.trim())162                                    let image_url = item.querySelector("img").src163                                    if (!image_url.match(/^https?:\/\//g)) image_url = "https:" + image_url164                                    if (image_url.match('data:image')) {165                                        debugger;166                                        console.log(item) // TODO: WTF IS THIS? Do we ignore the image, and thus use the placeholder image?167                                    }168                                    let instock = item.querySelector(".stock-level-wrapper") ? "c-false" : "" // If present, out of stock169                                    products.push({170                                        name: childName, price, brand, link, model_ID, image_url, instock,171                                        location: "DELL Store"172                                    })173                                }174                                return products175                            } catch (e) {176                                console.log(e)177                                return e178                            }179                        })180                        DELLProducts.push(...products)181                    })182                    await cluster.idle()183                    await cluster.close()184                    // Write to file185                    fs.writeFile(`${process.cwd()}/data/raw/monitors/dell.json`, JSON.stringify(DELLProducts), (err, file) => {186                        if (err) console.log(err)187                    })188                    console.log("PUPPETEER: Completed scraping Dell Store")189                    return true190                } catch (e) {191                    console.log(e)192                }193            }194            /* SECTION FOR HP Store */195            async function hp() {196                try {197                    console.log("PUPPETEER: Scraping HP Store")198                    const hpPage = await browser.newPage()199                    await hpPage.goto(links['HP Store'][0], { waitUntil: "networkidle2", timeout: pageTimeout })200                    let pages = 1;201                    try { 202                        pages = await hpPage.$eval(("#category\\.product\\.list > div:nth-child(3) > div.pages > ul"), e => e.childElementCount - 1)203                    } catch (e) {204                        // no element, means that there's only one page205                    }206                    console.log("FOUND " + pages + " pages")207                    await hpPage.close()208                    const cluster = await Cluster.launch(CLUSTEROPTS);209                    cluster.on('taskerror', (err, data, willRetry) => {210                        if (willRetry) {211                            console.warn(`Encountered an error while crawling ${data}. ${err.message}\nThis job will be retried`);212                        } else {213                            console.error(`Failed to crawl ${data}: ${err.message}`);214                        }215                    });216                    for (let i = 1; i <= pages; i++) {217                        let url = `${links['HP Store'][0]}&p=${i}`218                        cluster.queue({219                            url, i220                        })221                    }222                    const HPProducts = []223                    await cluster.task(async ({ page, data }) => {224                        try {225                            let url = data.url226                            let i = data.i227                            await page.exposeFunction('cleaner', cleaner)228                            console.log("Scraping page " + i + " of " + pages)229                            await page.goto(url, { waitUntil: 'networkidle2', timeout: pageTimeout })230                            let products = await page.evaluate(async () => {231                                try {232                                    let items = document.querySelectorAll(".product-item")233                                    let products = []234                                    for (let j = 0; j < items.length; j++) {235                                        let item = items[j]236                                        let childName = item.querySelector(".product-item-link").innerText237                                        let price = item.querySelector('.price').innerText238                                        let brand = "HP"239                                        let link = item.querySelector(".product-item-link").getAttribute("href")240                                        let model_ID = await cleaner(childName)241                                        let image_url = item.querySelector(".product-image-photo").dataset.src ? item.querySelector(".product-image-photo").dataset.src : item.querySelector(".product-image-photo").src242                                        let instock = item.querySelector(".out-stock-messaging") ? "c-false" : ""243                                        products.push({244                                            name: childName, price, brand, link, model_ID, image_url, instock,245                                            location: "HP Store"246                                        })247                                    }248                                    return products249                                } catch (e) {250                                    console.log(e)251                                    return e252                                }253                            })254                            console.log("Finished scraping page")255                            HPProducts.push(...products)256                            return true257                        } catch (e) {258                            console.log(e)259                            return false260                        }261                    })262                    await cluster.idle()263                    await cluster.close()264                    // console.log(ASUSProducts)265                    // Write to file266                    fs.writeFile(`${process.cwd()}/data/raw/monitors/hp.json`, JSON.stringify(HPProducts), (err, file) => {267                        if (err) console.log(err)268                    })269                    console.log("PUPPETEER: Completed scraping HP Store")270                    return true271                } catch (e) {272                    console.log(e)273                }274            }275            /* SECTION FOR LENOVO */276            async function lenovo() {277                try {278                    console.log("PUPPETEER: Scraping Lenovo Store")279                    const lenovoPage = await browser.newPage()280                    await lenovoPage.goto(links['LENOVO Store'][0], { waitUntil: 'networkidle2' })281                    let loadTimes = await lenovoPage.$eval(("#facetTop-count"), e => Math.ceil(Number(e.innerText.split(" ")[0]) / 20) - 1)282                    for (let i = 0; i < loadTimes; i++) {283                        console.log("Clicking load more button (" + (i + 1) + "/" + loadTimes + ")")284                        await lenovoPage.$eval((".loadmore-button"), e => e.click())285                        // await lenovoPage.waitForTimeout({waitUntil: "networkidle2"})286                        await timeout(1000)287                        let loading = true288                        while (loading) {289                            // Loading modal with class .searchLoader appears when loading (display: block)290                            loading = await lenovoPage.$eval((".searchLoader"), e => {291                                return e.style.display == "none" ? false : true292                            })293                            console.log("Waiting for loading...")294                            await timeout(1000)295                        }296                    }297                    console.log("All items loaded")298                    // await lenovoPage.exposeFunction("cleaner", cleaner)299                    let products = await lenovoPage.evaluate(() => {300                        try {301                            let items = document.querySelectorAll(".facetResults-item  ")302                            let products = []303                            for (let i = 0; i < items.length; i++) {304                                let item = items[i]305                                // Skip this if the price is NOT in the correct format306                                if (!item.querySelector(".xblodSize")) continue307                                let childName = item.querySelector('div.item-descriptions > a').innerText.replace(/\\n/g, "")308                                let price = item.querySelector(".xblodSize").innerText309                                let brand = "LENOVO"310                                let link = item.querySelector("div.item-descriptions > a").getAttribute("href")311                                link = 'https://www.lenovo.com' + link312                                let model_ID = item.dataset.productcode313                                let image_url = item.querySelector("div.image-container > a > img").dataset.original314                                if (!image_url.match(/^https?:\/\//g)) image_url = "https:" + image_url315                                if (image_url == "https:") image_url = "/images/missing.jpg"316                                let instock = item.dataset.productstatus != 'available' ? "c-false" : ""317                                let customizable = item.querySelector("button[type='submit']").innerText.toUpperCase().includes("CUSTOMIZE") ? true : false318                                products.push({319                                    name: childName, price, brand, link, model_ID, image_url, instock, customizable,320                                    location: "LENOVO Store"321                                })322                            }323                            console.log(products)324                            return products325                        } catch (e) {326                            console.log(e)327                            return e328                        }329                    })330                    await lenovoPage.close()331                    console.log(products)332                    // Write to file333                    fs.writeFile(`${process.cwd()}/data/raw/monitors/lenovo.json`, JSON.stringify(products), (err, file) => {334                        if (err) console.log(err)335                    })336                    console.log("PUPPETEER: Completed scraping LENOVO Store")337                    return true338                } catch (e) {339                }340            }341            /* SECTION FOR RAZER */342            async function razer() {343                try {344                    console.log("PUPPETEER: Scraping RAZER Store")345                    const razerPage = await browser.newPage()346                    await razerPage.goto(links['RAZER Store'][0], { waitUntil: "networkidle2" })347                    await razerPage.waitForTimeout(2500)348                    // await razerPage.exposeFunction('cleaner', cleaner)349                    let products = await razerPage.evaluate(async () => {350                        try {351                            let items = document.querySelectorAll(".grid-item")352                            let products = []353                            for (let i = 0; i < items.length; i++) {354                                let item = items[i]355                                let childName = item.querySelector(".product-item-title").innerText356                                let price = item.querySelector(".final-price").innerText357                                let brand = "RAZER"358                                let link = item.querySelector(".product-item-title").getAttribute("href")359                                link = "https://www.razer.com" + link360                                let model_ID = item.querySelector(".product-item-title").dataset['productId']361                                let image_url = item.querySelector(".cx-product-image > img").src362                                let instock = item.querySelector(".gtm_notify_me") ? "c-false" : ""363                                let customizable = false364                                products.push({365                                    name: childName, price, brand, link, model_ID, image_url, instock, customizable,366                                    location: "RAZER Store"367                                })368                            }369                            return products370                        } catch (e) {371                            console.log(e)372                            return e373                        }374                    })375                    await razerPage.close()376                    fs.writeFile(`${process.cwd()}/data/raw/laptops/razer.json`, JSON.stringify(products), (err, file) => {377                        if (err) console.log(err)378                    })379                    console.log("PUPPETEER: Completed scraping RAZER Store")380                    return true381                } catch (e) {382                    console.log(e)383                }384            }385            /* SECTION FOR GAIN CITY */386            async function gain() { // Confidence: 3387                console.log("PUPPETEER: Scraping Gain City")388                const gainPage = await browser.newPage()389                await gainPage.goto(links["Gain City"][0], { waitUntil: 'networkidle2', timeout: pageTimeout })390                await gainPage.exposeFunction("cleaner", cleaner)391                // Gain city automatically loads more items if we scroll down, so run a function to scroll all the way down until we can't anymore392                console.log("PUPPEETER: Scrolling page")393                gainPage.on('console', consoleObj => console.log(consoleObj.text()));394                await gainPage.evaluate(async () => {395                    // Scroll to bottom function        396                    const delay = 2000;397                    const wait = (ms) => new Promise(res => setTimeout(res, ms));398                    const scrollDown = async () => {399                        // Scroll to footer, which is always at the bottom400                        document.querySelectorAll(".product-item-info")[document.querySelectorAll(".product-item-info").length - 1].scrollIntoView({ behavior: 'smooth', block: 'end', inline: 'end' })401                    }402                    const run = async () => {403                        let reachedEnd = false404                        do {405                            console.log("Checking data")406                            let reachedEndElem = document.querySelector(".ias-noneleft")407                            console.log(reachedEndElem ? true : false)408                            if (reachedEndElem) reachedEnd = true409                            console.log("Scrolling")410                            await scrollDown();411                            await wait(delay);412                        } while (!reachedEnd);413                        await wait(delay);414                        return415                    }416                    await run()417                })418                console.log("PUPPEETER: Page completed scrolling")419                let gainData = await gainPage.evaluate(async () => {420                    let items = document.querySelectorAll(".product-item-info")421                    let products = []422                    for (item of items) {423                        let childName = item.querySelector(".product-item-link").innerText424                        if (!childName) childName = "MISSING INFO"425                        console.log(childName)426                        let price = item.querySelector(".price").innerText427                        let brand = childName.split(" ")[0]428                        let model_ID = await cleaner(item.querySelector(".product-model-number").innerText)429                        console.log(childName, " - ", model_ID)430                        let link = item.querySelector(".product-item-link").getAttribute("href").match(/((http|ftp|https):\/\/)?([\w_-]+(?:(?:\.[\w_-]+)+))([\w.,@?^=%&:\/~+#-]*[\w@?^=%&\/~+#-])?/i)[0]431                        if (!link.match(/^https?:\/\//g)) link = "https://" + link432                        let image_url = item.querySelector(".product-image-photo").src433                        if (!image_url.match(/^https?:\/\//g)) image_url = "https://" + image_url434                        let instock = item.querySelector(".stock.unavailable") ? "c-false" : ""435                        products.push({436                            name: childName, price, brand, link, model_ID, image_url, instock,437                            location: "Gain City"438                        })439                    }440                    return products441                })442                await gainPage.close()443                // Write to file444                fs.writeFile(`${process.cwd()}/data/raw/monitors/gain.json`, JSON.stringify(gainData), (err, file) => {445                    if (err) console.log(err)446                })447                console.log("PUPPETEER: Completed scraping GAIN CITY")448                return true449            }450            /* SECTION FOR HARVEY NORMAN */451            async function harvey() {452                try {453                    console.log("PUPPETEER: Scraping Harvey Norman")454                    // Create new tab455                    const harveyPage = await browser.newPage()456                    // Go to harvey norman laptop page457                    await harveyPage.goto(links["Harvey Norman"][0], { waitUntil: 'domcontentloaded', timeout: pageTimeout })458                    // Get the number of pages (Total / 20)459                    let harveyPages = await harveyPage.evaluate(() => Math.ceil(Number(document.querySelector("#pagination_contents > div.toolbar > div > div:nth-child(2) > div > div.pagination-amount.col-xs-4").innerText.split(" ")[0]) / 20))460                    console.log("PUPPEETER: Found pages: " + harveyPages)461                    await harveyPage.close()462                    const cluster = await Cluster.launch(CLUSTEROPTS);463                    cluster.on('taskerror', (err, data, willRetry) => {464                        if (willRetry) {465                            console.warn(`Encountered an error while crawling ${data}. ${err.message}\nThis job will be retried`);466                        } else {467                            console.error(`Failed to crawl ${data}: ${err.message}`);468                        }469                    });470                    console.log("After launching cluster")471                    await cluster.task(async ({ page, data }) => {472                        try {473                            let url = data.url474                            let i = data.i475                            console.log("PUPPEETER-CLUSTER: Scraping page " + i + " of " + harveyPages)476                            await page.goto(url, { waitUntil: 'networkidle2', timeout: pageTimeout })477                            let products = await page.evaluate(() => {478                                let items = document.querySelector(".col-xs-12.col-sm-9.col-md-9.omega").querySelectorAll('form')479                                let products = []480                                items.forEach(item => {481                                    let childName = item.querySelector(".product-info > a").getAttribute("title").trim().toUpperCase()482                                    if (!childName) childName = "MISSING INFO"483                                    let price = item.querySelector(".price").innerText.trim().toUpperCase()484                                    let brand = childName.split(" ")[0]485                                    let link = item.querySelector(".product-info > a").getAttribute("href").match(/((http|ftp|https):\/\/)?([\w_-]+(?:(?:\.[\w_-]+)+))([\w.,@?^=%&:\/~+#-]*[\w@?^=%&\/~+#-])?/i)[0]486                                    if (!link.match(/^https?:\/\//g)) link = "https://" + link487                                    let image_url = item.querySelector(".product-image > a > img").src488                                    if (!image_url.match(/^https?:\/\//g)) image_url = "https://" + image_url489                                    let instock = item.querySelector(".out-of-stock-block") ? "c-false" : ""490                                    products.push({491                                        name: childName, brand, price, link, image_url, instock,492                                        location: "Harvey Norman"493                                    })494                                })495                                return products496                            })497                            for (let j = 0; j < products.length; j++) {498                                console.log("PUPPEETER-CLUSTER: Scraping model " + (j + 1) + " of " + products.length + " (Page " + i + ")")499                                let product = products[j]500                                // Query the product page to get the model data501                                await page.goto(product.link, { waitUntil: "networkidle2", timeout: pageTimeout })502                                let model_ID = await page.$eval("#content_features > div > table:nth-child(3)", table => {503                                    let headings = Array.from(table.querySelectorAll('tr > th'))504                                    let index;505                                    headings.forEach(heading => {506                                        if (heading.innerText.toUpperCase() == "MODEL") index = headings.indexOf(heading) + 1507                                    })508                                    return table.querySelector(`tr:nth-child(${index}) > td`).innerText.toUpperCase().trim()509                                })510                                product.model_ID = cleaner(model_ID)511                            }512                            HARVEYPRODUCTS.push(...products)513                            return true514                        } catch (e) {515                            console.log(e)516                            return false517                        }518                    })519                    let HARVEYPRODUCTS = []520                    for (let i = 1; i <= harveyPages; i++) {521                        let url = `${links["Harvey Norman"][0]}page-${i}/`522                        // console.log("queueing" + url)523                        console.log("QUEUEING: ", url)524                        cluster.queue({ url, i })525                    }526                    await cluster.idle()527                    await cluster.close()528                    fs.writeFile(`${process.cwd()}/data/raw/monitors/harvey.json`, JSON.stringify(HARVEYPRODUCTS), (err, file) => {529                        if (err) console.log(err)530                    })531                    console.log("PUPPEETER: Completed scraping Harvey Norman")532                    return true533                } catch (e) {534                    console.log(e)535                    return e536                }537            }538            /* SECTION FOR CHALLENGER (using their API (AXIOS)) */539            async function challenger() { // MODEL CONFIDENCE : HIGH540                try {541                    console.log("AXIOS: Scraping Challenger")542                    // Get the object of every item ID in challenger's laptop database543                    const products = []544                    for (link of links["Challenger"]) {545                        let response = await axios.post("https://6bc318ijnf-dsn.algolia.net/1/indexes/*/queries?x-algolia-agent=Algolia%20for%20JavaScript%20(4.10.3)%3B%20Browser%20(lite)%3B%20instantsearch.js%20(4.25.2)%3B%20Vue%20(2.6.14)%3B%20Vue%20InstantSearch%20(3.8.1)%3B%20JS%20Helper%20(3.5.4)&x-algolia-api-key=88c8a34f2b7653f93b1ce0053dbc64fe&x-algolia-application-id=6BC318IJNF",546                            link,547                            {548                                "headers": {549                                    "content-type": "application/json"550                                }551                            }552                        )553                        let results = response.data.results554                        // console.log(results)555                        for (let r = 0; r < results.length; r++) {556                            let hits = results[r].hits557                            for (let i = 0; i < hits.length; i++) {558                                let item_ID = hits[i].item_id559                                let productResponse = await axios.get(`https://www.hachi.tech/product/${item_ID}/details`)560                                let data = productResponse.data.data561                                console.log(data)562                                if (!data) continue563                                let name = data.short_desc.toUpperCase().trim()564                                let price = `$${data.prices.regular_price}`565                                let brand = data.settings.dimensions.brand_id.toUpperCase().trim()566                                let link = `https://www.hachi.tech/product/${item_ID}`567                                let model_ID = data.settings.dimensions.model_id.toUpperCase().trim()568                                // EXCEPTION: ACER PRODUCTS - CHALLENGER PLACES THE ID WRONGLY, SO WE NEED TO USE THE PRODUCT NAME INSTEAD569                                if (brand == "ACER") model_ID = cleaner(name)570                                let image_url = data.image_name571                                if (!image_url.match(/^https?:\/\//g)) image_url = "https://" + image_url572                                await timeout(500)573                                let stockStatus = await axios.get(`https://www.hachi.tech/product/${item_ID}/inventory`)574                                let stock = stockStatus.data.data575                                let instock = "c-true"576                                if (!stock['delv_options'].length && !stock['scl_options'].length) instock = "c-false"577                                products.push({578                                    name,579                                    price,580                                    brand,581                                    model_ID,582                                    link,583                                    location: "Challenger",584                                    image_url,585                                    instock586                                })587                                console.log(`AXIOS: Scraping model ${i + 1} of ${hits.length} (Page ${r + 1}/${results.length})`)588                                await delay(500)589                            }590                        }591                    }592                    console.log("AXIOS: Completed scraping Challenger")593                    fs.writeFile(`${process.cwd()}/data/raw/monitors/challenger.json`, JSON.stringify(products), (err, file) => {594                        if (err) console.log(err)595                    })596                    return true597                } catch (e) {598                    console.log(e)599                    throw e600                }601            }602            /* SECTION FOR BEST DENKI */603            async function best() {604                console.log("PUPPETEER-CLUSTER: Scraping Best Denki")605                let BESTPRODUCTS = []606                const bestPage = await browser.newPage()607                const cluster1 = await Cluster.launch(CLUSTEROPTS);608                await cluster1.task(async ({ page, data }) => {609                    try {610                        let url = data.url611                        let i = data.i612                        console.log("PUPPETEER-CLUSTER (1): Scraping page " + (i + 1) + " of " + pages)613                        await page.goto(url, { waitUntil: 'networkidle2', timeout: pageTimeout })614                        await page.exposeFunction("cleaner", cleaner)615                        let products = await page.evaluate(async () => {616                            try {617                                let items = document.querySelectorAll(".product-item")618                                let products = []619                                for (let i = 0; i < items.length; i++) {620                                    let item = items[i]621                                    let instock = item.querySelector("button[type='submit']") ? "" : "c-false"622                                    // TODO: Best Denki lists everything (6XX items listed) but only a few are in stock (1XX in stock)623                                    // So, we ignore if it's out of stock624                                    // Special case625                                    if (instock == "c-false") continue626                                    var childName = item.querySelector(".product-item-link").innerText.trim()627                                    var price = item.querySelector(".custominhouseprice > .price").innerText.trim()628                                    let brand = item.querySelector(".product-item-brand").childName.split(" ")[0].toUpperCase()629                                    let link = item.querySelector(".product-item-link").getAttribute("href").match(/((http|ftp|https):\/\/)?([\w_-]+(?:(?:\.[\w_-]+)+))([\w.,@?^=%&:\/~+#-]*[\w@?^=%&\/~+#-])?/i)[0]630                                    if (!link.match(/^https?:\/\//g)) link = "https://" + link631                                    let model_ID = await cleaner(childName) // the model is the ending of the url632                                    let image_url = item.querySelector('.product-image-photo').dataset.src633                                    if (!image_url.match(/^https?:\/\//g)) image_url = "https://" + image_url634                                    products.push({635                                        name: childName, price, brand, link, image_url, model_ID, instock,636                                        location: "Best Denki"637                                    })638                                }639                                return products640                            } catch (e) {641                                console.log(e)642                                return e643                            }644                        })645                        BESTPRODUCTS.push(...products)646                    } catch (e) {647                        console.log(e)648                        return e649                    }650                })651                let pages = 0652                for (let i = 1; i < 10000; i++) {653                    let url = `${links["Best Denki"][0]}?p=${i}`654                    await bestPage.goto(url, { waitUntil: 'domcontentloaded' })655                    let ended = await bestPage.$('#maincontent > div.row > div.column.main.col-lg-9.col-md-9.col-sm-12.col-xs-12.pull-right > div.message.info.empty')656                    if (ended) {657                        pages = i - 1658                        break;659                    }660                }661                for (let i = 1; i < pages; i++) {662                    let url = `${links["Best Denki"][0]}?p=${i}`663                    cluster1.queue({664                        url, i665                    })666                }667                console.log("PUPPETEER: Found pages: " + pages)668                await bestPage.close()669                await cluster1.idle()670                await cluster1.close()671                fs.writeFile(`${process.cwd()}/data/raw/monitors/best.json`, JSON.stringify(BESTPRODUCTS), (err, file) => {672                    if (err) console.log(err)673                })674                console.log("PUPPETEER-CLUSTER: Completed scraping Best Denki")675                return true676            }677            /* SECTION FOR COURTS */678            async function courts() {679                console.log("PUPPETEER-CLUSTER: Scraping Courts")680                const courtsPage = await browser.newPage()681                await courtsPage.goto(links["Courts"][0], { waitUntil: "networkidle2", timeout: pageTimeout })682                // Find out how many pages there are683                // Total products divided by # per page (32)684                let courtsPages = await courtsPage.evaluate(() => Math.ceil(Number(document.querySelector('.product-count').innerText.split(" ")[0]) / 32))685                console.log("PUPPETEER: Found pages: " + courtsPages)686                await courtsPage.close()687                const cluster = await Cluster.launch(CLUSTEROPTS);688                await cluster.task(async ({ page, data }) => {689                    try {690                        page.on('console', async msg => {691                            const args = await msg.args()692                            args.forEach(async (arg) => {693                                const val = await arg.jsonValue()694                                // value is serializable695                                if (JSON.stringify(val) !== JSON.stringify({})) console.log(val)696                                // value is unserializable (or an empty oject)697                                else {698                                    const { type, subtype, description } = arg._remoteObject699                                    console.log(`type: ${type}, subtype: ${subtype}, description:\n ${description}`)700                                }701                            })702                        });703                        let url = data.url704                        let i = data.i705                        console.log("PUPPETEER-CLUSTER: Scraping page " + i + " of " + courtsPages)706                        await page.goto(url, { waitUntil: 'networkidle2', timeout: pageTimeout })707                        console.log("Page Loaded!")708                        await page.exposeFunction("cleaner", cleaner)709                        console.log("Function exposed!")710                        let products = await page.evaluate(async () => {711                            try {712                                let items = Array.from(document.querySelector(".columns").querySelector(".product-items").querySelectorAll(".product-item-info"))713                                let products = []714                                for (item of items) {715                                    // console.log("Here")716                                    let childName = item.querySelector(".product-item-name").innerText.trim().toUpperCase()717                                    if (!childName) childName = "MISSING INFO"718                                    // console.log("Here2")719                                    // First word is the brand720                                    let brand = childName.split(" ")[0]721                                    // console.log("Her3e")722                                    let price = item.querySelector(".price").innerText.trim().toUpperCase()723                                    // console.log("Here4")724                                    let link = item.querySelector(".product-item-name > a").getAttribute("href").match(/((http|ftp|https):\/\/)?([\w_-]+(?:(?:\.[\w_-]+)+))([\w.,@?^=%&:\/~+#-]*[\w@?^=%&\/~+#-])?/i)[0]725                                    if (!link.match(/^https?:\/\//g)) link = "https://" + link726                                    let model_ID = await cleaner(childName) ? await cleaner(childName) : childName727                                    // console.log("Here5")728                                    let image_url = item.querySelector(".product-image-photo").getAttribute("data-original")729                                    if (!image_url.match(/^https?:\/\//g)) image_url = "https://" + image_url730                                    // console.log("Here6")731                                    let instock = item.querySelector(".stock.unavailable") ? "c-false" : ""732                                    // console.log("Here7")733                                    products.push({734                                        name: childName, brand, price, link, model_ID, image_url, instock,735                                        location: "Courts"736                                    })737                                    // console.log("Here8")738                                }739                                return products740                            } catch (e) {741                                console.log(e)742                                return e743                            }744                        })745                        console.log(products)746                        COURTSPRODUCTS.push(...products)747                    } catch (e) {748                        console.log(e)749                        return e750                    }751                })752                let COURTSPRODUCTS = []753                for (let i = 1; i <= courtsPages; i++) {754                    let url = `${links["Courts"][0]}&p=${i}`755                    console.log("Queuing: " + i + " of " + courtsPages)756                    cluster.queue({ url, i })757                }758                await cluster.idle()759                await cluster.close()760                fs.writeFile(`${process.cwd()}/data/raw/monitors/courts.json`, JSON.stringify(COURTSPRODUCTS), (err, file) => {761                    if (err) console.log(err)762                })763                console.log("PUPPETEER-CLUSTER: Completed scraping Courts")764                return true765            }766            // /* SECTION FOR LG */767            // async function lg() {768            //     console.log("PUPPETEER: SCRAPING LG Store")769            //     const lgPage = await browser.newPage()770            //     await lgPage.goto(links["LG Store"][0], { waitUntil: "networkidle2", timeout: pageTimeout })771            //     await lgPage.exposeFunction("cleaner", cleaner)772            //     let products = await lgPage.evaluate(async () => {773            //         let products = []774            //         let items = document.querySelector()775            //         let childName = 776            //         products.push({777            //             name: childName, brand, price, link, model_ID, image_url, instock,778            //             location: "Courts"779            //         })780            //         return products781            //     })782            // }783            await browser.close()784            function cleaner(string) {785                var original = string786                var string = string.trim().toLowerCase()787                // Replace all non-breaking space and underscores788                string = string.replace(/Â |_/gi, " ")789                var finalArr = []790                // 3) Remove all special characters791                string = string.replace(/,|Â®|â¢|\+|\/\/|â|:/g, "")792                // 1) Remove remove all '-' that have spaces on either side - e.g. 793                // remove ' -', '- ', ' - ' but not 'adsfads-asdfasd'794                string = string.replace(/[- ][ -]/g, " ")795                // 2) Remove specifications (RAM / SSD)796                // string = string.replace(/(\s\d+[kgtp]b)+(?=\W)|^\d+[kgtb]b(?=\W)|\s\d+[kgtb]b/gi, "")797                string = string.replace(/\d+[gt]b/gi, "")798                string = string.replace(/\d\d\dSSD/gi, "")799                // GHZ800                string = string.replace(/\d.\d[gm]hz/gi, "")801                // 8-core802                string = string.replace(/\d-core/gi, "")803                // cpu name804                string = string.replace(/[ir]\d[-\s]\d\d\d\d\w?\w?[kqe]?/gi, "")805                string = string.replace(/ryzen\s\d\s\d\d\d\d(\d?|(\d\d)?)\w/gi, "")806                // GPU807                string = string.replace(/(R|G)TX\s?\d?\d\d\d(TI)?|(MX\s?\d\d\d)|RX\s?\d\d\d\d\s?(XT)?/gmi, "")808                // 4) remove pure alphanumerics?809                // 5) Remove sizing, either 810                // NN-inch, NNinch, NN-in, NNin, NN', NN"811                string = string.replace(/((\d\d\.)?\d+-?(in(ch)?|"|''?))/gi, "")812                // 1) Usually, stuff in (text)/[text]/*text*/ denotes additional information, like what challenger does.813                // Harvey norman puts the model number inside the () though, so what we do is we check each []/()/**, if 814                // inside has at least ONE word as defined in dictionary.json, then we remove the whole string, else, 815                // we only remove the brackets816                var foundBracket = string.match(/\[[^\]]*]|\([^)]*\)*|\*[^*]*\*/g)817                if (foundBracket) {818                    foundBracket.forEach(match => {819                        // Strip the brakcets for word comparison820                        match = match.replace(/\(|\)|\[|\]|\*/g, "")821                        var wordArr = match.split(" ")822                        for (var i = 0; i < wordArr.length; i++) {823                            var word = wordArr[i]824                            var toRemove = false825                            if (dictionary[word] || extras[word]) {826                                // Remove this bracket827                                toRemove = true828                                break;829                            }830                        }831                        if (toRemove) {832                            // // This bracket (the 'match') is to be removed833                            // string.replace(match, " ")834                        } else {835                            // Don't remove it, but strip the brackets836                            finalArr.push(match)837                        }838                    })839                }840                // BRACKETS HAVE BEEN DEALT WITH, now REMOVE ALL THE BRACKETS841                string = string.replace(/\[[^\]]*]|\([^)]*\)*|\*[^*]*\*/g, " ")842                // console.log("*****************")843                // console.log(string )844                // console.log(original)845                // console.log("*****************")846                // remove all '-' that have spaces on either side - e.g. 847                // remove ' -', '- ', ' - ' but not 'adsfads-asdfasd'848                string = string.replace(/[- ][ -]/g, " ")849                // Final remove special chars850                string = string.replace(/,|Â®|â¢|\+|\/\/|â|:/g, "")851                // Remove lone numbers852                string = string.replace(/(?<=[ \t])(\d+)(?=[ \t])/g, "")853                // Split into array854                var stringArr = string.toLowerCase().split(" ")855                // for each word, check if exists in dictionary or extras856                let cleanedArr = []857                for (var i = 0; i < stringArr.length; i++) {858                    var word = stringArr[i]859                    var toRemove = false860                    if (dictionary[word] || extras[word]) {861                        // to be removed862                        continue863                    }864                    if (!Number.isNaN(Number(word))) {865                        // if the word is a number, we don't add it866                        continue867                    }868                    // cleanedArr.push(word)869                    finalArr.push(word)870                }871                return finalArr.join(" ").trim().toUpperCase()872            }873            console.log("----------------- COMPLETED EXECUTING FILE: scraper.js -----------------")874            require(`${__dirname}/sorter.js`)875        } catch (e) {876            console.log(e)877        }878    })();879function timeout(ms) {880    return new Promise(resolve => setTimeout(resolve, ms));...

puppeteer-run.js

Source:puppeteer-run.js

...44    const page3 = await browser.newPage();45    // dá»nh dáº¡ng khung cá»a sá» hiá»n thá»46   // page.setViewport({ width: 1280, height: 926 });47   try{48puppeteer.clearCustomQueryHandlers();49  await page3.goto('http://kenh14fsdfsdfds.vn/', {timeout: 5000});50 51   }catch(error){52       console.log("failed to open the page");53   }54*/55    const page = await browser.newPage();56    // dá»nh dáº¡ng khung cá»a sá» hiá»n thá»57   // page.setViewport({ width: 1280, height: 926 });58   try{59 //   puppeteer.clearCustomQueryHandlers();60await page.goto('http://kenh14.vn/',{waitUntil: 'networkidle2', timeout: 0});61 // Wait 5 seconds62 //await new Promise(resolve => setTimeout(resolve, 5000));63   // console.log('Page loaded');64    65   }catch(error){66       console.log("failed to open the page");67   }68 //finally {69    // Always close the browser70   // await browser.close();71//}72   73   ...

sidebars.js

Source:sidebars.js

1module.exports = {2    docs: {3        Puppeteer: [4            "puppeteer.puppeteer",{5            Methods: [6            "puppeteer.puppeteer.clearcustomqueryhandlers",7            'puppeteer.puppeteer.connect',8            'puppeteer.puppeteer.customqueryhandlernames',9            'puppeteer.puppeteer.devices',10            'puppeteer.puppeteer.errors',11            'puppeteer.puppeteer.networkconditions',12            'puppeteer.puppeteer.registercustomqueryhandler',13            'puppeteer.puppeteer.unregistercustomqueryhandler',14            ]15            }16        ],17        "BrowserFetcher": [18            'puppeteer.browserfetcher',19            'puppeteer.browserfetcher.candownload',20            'puppeteer.browserfetcher.download',21            'puppeteer.browserfetcher.host',22            'puppeteer.browserfetcher.localrevisions',23            'puppeteer.browserfetcher.platform',24            'puppeteer.browserfetcher.product',25            'puppeteer.browserfetcher.remove',26            'puppeteer.browserfetcher.revisioninfo',27        ],28        "Browser": [29            'puppeteer.browser',30        'puppeteer.browser.browsercontexts',31        'puppeteer.browser.close',32        'puppeteer.browser.createincognitobrowsercontext',33        'puppeteer.browser.defaultbrowsercontext',34        'puppeteer.browser.disconnect',35        'puppeteer.browser.isconnected',36        'puppeteer.browser.newpage',37        'puppeteer.browser.pages',38        'puppeteer.browser.process',39        'puppeteer.browser.target',40        'puppeteer.browser.targets',41        'puppeteer.browser.useragent',42        'puppeteer.browser.waitfortarget',43        'puppeteer.browser.wsendpoint',44        ],45        "BrowserContext": [46            'puppeteer.browsercontext',47        'puppeteer.browsercontext.browser',48        'puppeteer.browsercontext.overridepermissions',49        'puppeteer.browsercontext.close',50        'puppeteer.browsercontext.isincognito',51        'puppeteer.browsercontext.newpage',52        'puppeteer.browsercontext.overridepermissions',53        'puppeteer.browsercontext.pages',54        'puppeteer.browsercontext.targets',55        'puppeteer.browsercontext.waitfortarget',56        ],57        "WebWorker": [58            'puppeteer.webworker',59        'puppeteer.webworker.evaluate',60        'puppeteer.webworker.evaluatehandle',61        'puppeteer.webworker.executioncontext',62        'puppeteer.webworker.url',63        ],64        "Accessibility": [65            'puppeteer.accessibility',66        'puppeteer.accessibility.snapshot',67        ],68        "FileChooser": [69            'puppeteer.filechooser',70        'puppeteer.filechooser.accept',71        'puppeteer.filechooser.cancel',72        'puppeteer.filechooser.ismultiple',73        ],74        "ExecutionContext": [75            'puppeteer.executioncontext',76        'puppeteer.executioncontext.evaluate',77        'puppeteer.executioncontext.evaluatehandle',78        'puppeteer.executioncontext.frame',79        'puppeteer.executioncontext.queryobjects',80        ],81        "JSHandle": [82            'puppeteer.jshandle',83        'puppeteer.jshandle.aselement',84        'puppeteer.jshandle.dispose',85        'puppeteer.jshandle.evaluate',86        'puppeteer.jshandle.evaluatehandle',87        'puppeteer.jshandle.executioncontext',88        'puppeteer.jshandle.getproperties',89        'puppeteer.jshandle.getproperty',90        'puppeteer.jshandle.jsonvalue',91        ],92        "ElementHandle": [93            'puppeteer.elementhandle',94        'puppeteer.elementhandle._',95        'puppeteer.elementhandle.__',96        'puppeteer.elementhandle.__eval',97        'puppeteer.elementhandle._eval',98        'puppeteer.elementhandle._x',99        'puppeteer.elementhandle.aselement',100        'puppeteer.elementhandle.boundingbox',101        'puppeteer.elementhandle.boxmodel',102        'puppeteer.elementhandle.click',103        'puppeteer.elementhandle.contentframe',104        'puppeteer.elementhandle.focus',105        'puppeteer.elementhandle.hover',106        'puppeteer.elementhandle.clickablepoint',107        'puppeteer.elementhandle.drag',108        'puppeteer.elementhandle.draganddrop',109        'puppeteer.elementhandle.dragenter',110        'puppeteer.elementhandle.dragover',111        'puppeteer.elementhandle.drop',112        'puppeteer.elementhandle.isintersectingviewport',113        'puppeteer.elementhandle.press',114        'puppeteer.elementhandle.screenshot',115        'puppeteer.elementhandle.select',116        'puppeteer.elementhandle.tap',117        'puppeteer.elementhandle.type',118        'puppeteer.elementhandle.uploadfile',119        ],120        "HTTPRequest": [121            'puppeteer.httprequest',122        'puppeteer.httprequest.abort',123        'puppeteer.httprequest.continue',124        'puppeteer.httprequest.failure',125        'puppeteer.httprequest.frame',126        'puppeteer.httprequest.headers',127        'puppeteer.httprequest.isnavigationrequest',128        'puppeteer.httprequest.method',129        'puppeteer.httprequest.postdata',130        'puppeteer.httprequest.redirectchain',131        'puppeteer.httprequest.resourcetype',132        'puppeteer.httprequest.respond',133        'puppeteer.httprequest.response',134        'puppeteer.httprequest.url',135        ],136        "HTTPRespose": [137            'puppeteer.httpresponse',138        'puppeteer.httpresponse.buffer',139        'puppeteer.httpresponse.frame',140        'puppeteer.httpresponse.fromcache',141        'puppeteer.httpresponse.fromserviceworker',142        'puppeteer.httpresponse.headers',143        'puppeteer.httpresponse.json',144        'puppeteer.httpresponse.ok',145        'puppeteer.httpresponse.remoteaddress',146        'puppeteer.httpresponse.request',147        'puppeteer.httpresponse.securitydetails',148        'puppeteer.httpresponse.status',149        'puppeteer.httpresponse.statustext',150        'puppeteer.httpresponse.text',151        'puppeteer.httpresponse.url',152        ],153        "SecurityDetails": [154            'puppeteer.securitydetails',155        'puppeteer.securitydetails.issuer',156        'puppeteer.securitydetails.protocol',157        'puppeteer.securitydetails.subjectalternativenames',158        'puppeteer.securitydetails.subjectname',159        'puppeteer.securitydetails.validfrom',160        'puppeteer.securitydetails.validto',161        ],162        "Target": [163            'puppeteer.target',164        'puppeteer.target.browser',165        'puppeteer.target.browsercontext',166        'puppeteer.target.createcdpsession',167        'puppeteer.target.opener',168        'puppeteer.target.page',169        'puppeteer.target.type',170        'puppeteer.target.url',171        'puppeteer.target.worker',172        ],173        "CDPSession": [174            'puppeteer.cdpsession',175        'puppeteer.cdpsession.connection',176        'puppeteer.cdpsession.detach',177        'puppeteer.cdpsession.send',178        ],179        "Coverage": [180            'puppeteer.coverage',181        'puppeteer.coverage.startcsscoverage',182        'puppeteer.coverage.startjscoverage',183        'puppeteer.coverage.stopcsscoverage',184        'puppeteer.coverage.stopjscoverage',185        ],186        "TimeOutError": [187            'puppeteer.timeouterror'188        ],189        "EventEmitter": [190            'puppeteer.eventemitter',191        'puppeteer.eventemitter.addlistener',192        'puppeteer.eventemitter.emit',193        'puppeteer.eventemitter.addlistener',194        'puppeteer.eventemitter.listenercount',195        'puppeteer.eventemitter.off',196        'puppeteer.eventemitter.on',197        'puppeteer.eventemitter.once',198        'puppeteer.eventemitter.removelistener',199        'puppeteer.eventemitter.removealllisteners',200        ],201    },...

QueryHandler.js

Source:QueryHandler.js

...139exports.customQueryHandlerNames = customQueryHandlerNames;140/**141 * @internal142 */143function clearCustomQueryHandlers() {144    customQueryHandlerNames().forEach(unregisterCustomQueryHandler);145}146exports.clearCustomQueryHandlers = clearCustomQueryHandlers;147/**148 * @internal149 */150function getQueryHandlerAndSelector(selector) {151    const hasCustomQueryHandler = /^[a-zA-Z]+\//.test(selector);152    if (!hasCustomQueryHandler)153        return { updatedSelector: selector, queryHandler: _defaultHandler };154    const index = selector.indexOf('/');155    const name = selector.slice(0, index);156    const updatedSelector = selector.slice(index + 1);157    const queryHandler = _queryHandlers.get(name);...

Puppeteer.js

Source:Puppeteer.js

...125    }126    /**127     * Clears all registered handlers.128     */129    clearCustomQueryHandlers() {130        clearCustomQueryHandlers();131    }...

index.js

Source:index.js

...101}102// async function like(id) {103//   try {104//     const url = 'https://padlet.com/chithuong70/uahr2skx76gww651';105//     puppeteer.clearCustomQueryHandlers()106//     const browser = await puppeteer.launch()107//     const page = await browser.newPage()108//     await page.goto(url, {109//       waitUntil: 'networkidle2',110//     })111//     const work = await page.evaluate(async (id) => {112//       a = document.querySelectorAll(`#${id.id} section .cursor-pointer`)[0];113//       if (a) {114//         a.click()115//         b = document.querySelectorAll(`#${id.id} section .cursor-pointer span`)[0]116//         return b.innerText117//       }118//       return a119//     }, { id })120//     console.log(`work[${id}] =>`, work)121//     await browser.close()122//     await sleep(Math.floor(Math.random() * 1000))123//   } catch (error) {124//     console.log("error", error)125//   }126// }127// async function comment(id) {128//   try {129//     const url = 'https://padlet.com/chithuong70/uahr2skx76gww651';130//     puppeteer.clearCustomQueryHandlers()131//     const browser = await puppeteer.launch()132//     const page = await browser.newPage()133//     await page.goto(url)134//     const work = await page.evaluate(async () => {135//       a = document.querySelectorAll(`#${id} section .cursor-pointer`)[0];136//       if (a) {137//         return a.click()138//       }139//       return false140//     })141//     console.log("work =>", work)142//     await browser.close()143//     await sleep(Math.floor(Math.random() * 100))144//     console.log(i)...

calenderAPI.js

Source:calenderAPI.js

1//read heavy2// start time,   end time,   name 3const { clearCustomQueryHandlers } = require("puppeteer");4//insert events 5// return ordered collection ordered by start time, ascending, then end time ascending , then by name 6//Questions7// When you say read heavy, you just want the whole thing right?8//can i just assume a double for now, and replace with time9class Calendar {10    eventList;11    //constructor should initialize an empty list12    constructor(){13        this.eventList = [];14    }15    //insert an event 16    //Because it is read heavy, we are going to keep the list ordered17    insert(start, end, name){18        let newEvent = new Event(start, end, name);19    20        let added = false;21        for(let i=0; i<this.eventList.length; i++){22            let currentEvent = this.eventList[i];23            //traverse the list until we find that the next event start time is greater or equal to newEvent24            if(currentEvent.startTime >= newEvent.startTime){25                26                //if not equal, add before27                if(currentEvent.startTime != newEvent.startTime){28                    this.eventList.splice(i, 0, newEvent);29                    added = true;30                    break;31                }32                //start time is equal. compare the end time33                else if(currentEvent.endTime > newEvent.endTime){34                    console.log('new event end time is smaller')35                    this.eventList.splice(i, 0, newEvent);36                    added = true;37                    break;38                }39                else if(currentEvent.endTime == newEvent.endTime){40                    //check name41                    if(currentEvent.name >= newEvent.name){42                43                        this.eventList.splice(i, 0, newEvent);44                        added = true;45                    }46                    else{47                        continue;48                    }49                }50                //end time for event is less than new event -> go to next event51                else{52                    continue;53                }54            }55            56            57        }58        if(!added){59            this.eventList.push(newEvent)60            console.log('added')61        }62    }63    view(){64        return this.eventList;65    }66    delete(name){67        for(let i =0; i< this.eventList.length; i++){68            if(this.eventList[i].name == name){69                this.eventList.splice(i,1);70                return true;71            }72        }73        return false;74    }75    eventsForPage(pageSize, pageNum){76        let eventItemStart = ((pageNum-1)  * pageSize ) 77        78        if(eventItemStart >= this.eventList.length){79            return [];80        }81        else if(eventItemStart + pageSize >= this.eventList.length){82            pageSize = this.eventList.length  - eventItemStart;83        }84     85        return this.eventList.splice(eventItemStart, pageSize);86    }87  88}89class Event {90    startTime;91    endTime;92    name; 93    constructor(startTime, endTime, name){94        this.startTime = startTime;95        this.endTime = endTime;96        this.name = name;97    }98}99let calendar = new Calendar();100// calendar.insert(3, 8, 'a');101// calendar.insert(3,5, 'b');102// calendar.insert(4,6, 'c')103// console.log(calendar.view());104// console.log(calendar.delete('a'));105// console.log(calendar.view());106calendar.insert(1,2,'a');107console.log(calendar.view());108calendar.insert(3,4,'b');109calendar.insert(4,5,'c');110calendar.insert(6,7,'d');111calendar.insert(8,9,'e');112console.log(calendar.view());...

payments.js

Source:payments.js

1// const stripe = require("stripe")(process.env.STRIPE_SECRET_KEY);2const stripe = require("stripe")(3  "sk_test_51IhcBtCueTz3N4RSPKazGPYXmvuzwys5QazLWAgAWbNIqEngyHpl7bjfPBlsdUqGWhilPXrFFTYtVGJmH7YO8o7C00ZvDeQw8V"4);5const { clearCustomQueryHandlers } = require("puppeteer");6const uuid = require("uuid/v4");7const order = require("../models/order");8const user = require("../models/user");9const Payment = require("../models/payment");10const { OrderDetails } = require("./orders");11exports.orderPayment = async (req, res) => {12  const { products, token, order_id } = req.body;13  const idempotencyKey = uuid();14  //   console.log(token);15  try {16    const { userid, total } = (17      await order.findById(order_id, "userid total").exec()18    )._doc;19    const { email } = (await user.findById(userid, "email").exec())._doc;20    const customer = await stripe.customers.create({21      email: token.email,22      source: token.id,23    });24    const charge = await stripe.charges.create(25      {26        amount: total * 100,27        currency: "usd",28        customer: customer.id,29        receipt_email: token.email,30        description: `order payment for amount of ${total} done by the user with id ${userid} on ${new Date()}`,31      },32      { idempotencyKey }33    );34    console.log(charge);35    const _payment = new Payment({36      payment_id: charge.id,37      amount: charge.amount / 100,38      currency: charge.currency,39      stripe_customer_id: charge.customer,40      user_id: userid,41      payment_for: "order",42      order_deposit_id: order_id,43      description: charge.description,44      status: charge.outcome.network_status,45      card: {46        brand: charge.payment_method_details.card.brand,47        country: charge.payment_method_details.card.country,48        exp_month: charge.payment_method_details.card.exp_month,49        exp_year: charge.payment_method_details.card.exp_year,50        funding: charge.payment_method_details.card.funding,51      },52      email: charge.receipt_email,53      receipt_url: charge.receipt_url,54    });55    const payment_done = await _payment.save();56    if (payment_done) {57      return res.status(200).json({58        message: "Payment has been successfull",59      });60    }61  } catch (error) {62    console.log(error);63    return res.status(400).json({64      message: "There has been an error while processing the payment",65      error,66    });67  }...

Using AI Code Generation

1const puppeteer = require('puppeteer');2(async () => {3  const browser = await puppeteer.launch();4  const page = await browser.newPage();5  await page.setRequestInterception(true);6  page.on('request', request => {7    if (request.url().endsWith('.png') || request.url().endsWith('.jpg'))8      request.abort();9      request.continue();10  });11  await browser.close();12})();

Using AI Code Generation

1const puppeteer = require('puppeteer');2(async () => {3  const browser = await puppeteer.launch({headless: false});4  const page = await browser.newPage();5  await page.type('input[type="text"]', 'Hello World');6  await page.keyboard.press('Enter');7  await page.waitForNavigation();8  await page.waitForSelector('input[type="text"]');9  await page.type('input[type="text"]', 'Hello World');10  await page.keyboard.press('Enter');11  await page.waitForNavigation();12  await page.waitForSelector('input[type="text"]');13  await page.type('input[type="text"]', 'Hello World');14  await page.keyboard.press('Enter');15  await page.waitForNavigation();16  await page.waitForSelector('input[type="text"]');17  await page.type('input[type="text"]', 'Hello World');18  await page.keyboard.press('Enter');19  await page.waitForNavigation();20  await page.waitForSelector('input[type="text"]');21  await page.type('input[type="text"]', 'Hello World');22  await page.keyboard.press('Enter');23  await page.waitForNavigation();24  await page.waitForSelector('input[type="text"]');25  await page.type('input[type="text"]', 'Hello World');26  await page.keyboard.press('Enter');27  await page.waitForNavigation();28  await page.waitForSelector('input[type="text"]');29  await page.type('input[type="text"]', 'Hello World');30  await page.keyboard.press('Enter');31  await page.waitForNavigation();32  await page.waitForSelector('input[type="text"]');33  await page.type('input[type="text"]', 'Hello World');34  await page.keyboard.press('Enter');35  await page.waitForNavigation();36  await page.waitForSelector('input[type="text"]');37  await page.type('input[type="text"]', 'Hello World');38  await page.keyboard.press('Enter');39  await page.waitForNavigation();40  await page.waitForSelector('input[type="text"]');41  await page.type('input[type="text"]', 'Hello World');42  await page.keyboard.press('Enter');43  await page.waitForNavigation();44  await page.waitForSelector('input[type="text"]');45  await page.type('input[type="text"]', 'Hello World');46  await page.keyboard.press('Enter');47  await page.waitForNavigation();48  await page.waitForSelector('input[type="text"]');49  await page.type('input[type="text"]',

Using AI Code Generation

1const puppeteer = require('puppeteer');2(async () => {3  const browser = await puppeteer.launch();4  const page = await browser.newPage();5  await page.setRequestInterception(true);6  await page.on('request', (request) => {7    if (request.url().endsWith('.png') || request.url().endsWith('.jpg'))8      request.abort();9      request.continue();10  });11  await page.waitForSelector('img');12  await page.setRequestInterception(false);13  await page.waitFor(3000);14  await browser.close();15})();16const puppeteer = require('puppeteer');17(async () => {18  const browser = await puppeteer.launch();19  const page = await browser.newPage();20  await page.setRequestInterception(true);21  await page.on('request', (request) => {22    if (request.url().endsWith('.png') || request.url().endsWith('.jpg'))23      request.abort();24      request.continue();25  });26  await page.waitForSelector('img');27  await page.setRequestInterception(false);28  await page.waitFor(3000);29  await browser.close();30})();31const puppeteer = require('puppeteer');32(async () => {33  const browser = await puppeteer.launch();34  const page = await browser.newPage();35  await page.setRequestInterception(true);36  await page.on('request', (request) => {37    if (request.url().endsWith('.png') || request.url().endsWith('.jpg'))38      request.abort();39      request.continue();40  });41  await page.waitForSelector('img');42  await page.setRequestInterception(false);43  await page.waitFor(3000);44  await browser.close();45})();46const puppeteer = require('puppeteer');47(async () => {48  const browser = await puppeteer.launch();49  const page = await browser.newPage();50  await page.setRequestInterception(true);51  await page.on('request', (request) => {52    if (request.url().endsWith('.png') || request.url().endsWith('.jpg'))

Using AI Code Generation

1const puppeteer = require('puppeteer');2(async () => {3    const browser = await puppeteer.launch();4    const page = await browser.newPage();5    await page.setRequestInterception(true);6    await page.on('request', request => {7        console.log('Intercepted request: ' + request.url());8        request.continue();9    });10    await page.on('requestfinished', request => {11        console.log('Request finished: ' + request.url());12    });13    await page.on('requestfailed', request => {14        console.log('Request failed: ' + request.url());15    });16    await page.clearCustomQueryHandlers();17    await browser.close();18})();19const puppeteer = require('puppeteer');20(async () => {21    const browser = await puppeteer.launch();22    const page = await browser.newPage();23    await page.setRequestInterception(true);24    await page.on('request', request => {25        console.log('Intercepted request: ' + request.url());26        request.continue();27    });28    await page.on('requestfinished', request => {29        console.log('Request finished: ' + request.url());30    });31    await page.on('requestfailed', request => {32        console.log('Request failed: ' + request.url());33    });34    await page.setRequestInterception(false);35    await browser.close();36})();37const puppeteer = require('puppeteer');38(async () => {39    const browser = await puppeteer.launch();40    const page = await browser.newPage();41    await page.setRequestInterception(true);42    await page.on('request', request => {43        console.log('Intercepted request: ' + request.url());44        request.continue();45    });46    await page.on('request

Using AI Code Generation

1const puppeteer = require('puppeteer');2(async () => {3  const browser = await puppeteer.launch({ headless: false, args: ['--no-sandbox', '--disable-setuid-sandbox'] });4  const page = await browser.newPage();5  await page.setRequestInterception(true);6  page.on('request', request => {7    if (request.resourceType() === 'image')8      request.abort();9      request.continue();10  });11  await page.click('input[name="q"]');12  await page.keyboard.type('Puppeteer');13  await page.keyboard.press('Enter');14  await page.waitForSelector('div[id="search"]');15  await page.setRequestInterception(false);16  await browser.close();17})();18const puppeteer = require('puppeteer');19(async () => {20  const browser = await puppeteer.launch({ headless: false, args: ['--no-sandbox', '--disable-setuid-sandbox'] });21  const page = await browser.newPage();22  await page.setRequestInterception(true);23  page.on('request', request => {24    if (request.resourceType() === 'image')25      request.abort();26      request.continue();27  });28  await page.click('input[name="q"]');29  await page.keyboard.type('Puppeteer');30  await page.keyboard.press('Enter');31  await page.waitForSelector('div[id="search"]');32  await page.setRequestInterception(false);33  await browser.close();34})();35const puppeteer = require('puppeteer');36(async () => {37  const browser = await puppeteer.launch({ headless: false, args: ['--no-sandbox', '--disable-setuid-sandbox'] });38  const page = await browser.newPage();39  await page.setRequestInterception(true);40  page.on('request', request => {41    if (request.resourceType() === 'image')42      request.abort();43      request.continue();44  });45  await page.click('input

Using AI Code Generation

1const puppeteer = require('puppeteer');2(async () => {3  const browser = await puppeteer.launch();4  const page = await browser.newPage();5  await page.evaluate(() => {6    window.customQueryHandlers = {7      custom: () => document.querySelector('div'),8    };9  });10  await page.evaluate(() => {11    window.customQueryHandlers = {};12  });13  await browser.close();14})();15const puppeteer = require('puppeteer');16(async () => {17  const browser = await puppeteer.launch();18  const page = await browser.newPage();19  await page.evaluate(() => {20    window.customQueryHandlers = {21      custom: () => document.querySelector('div'),22    };23  });24  await page.evaluate(() => {25    delete window.customQueryHandlers;26  });27  await browser.close();28})();29const puppeteer = require('puppeteer');30(async () => {31  const browser = await puppeteer.launch();32  const page = await browser.newPage();33  await page.evaluate(() => {34    window.customQueryHandlers = {35      custom: () => document.querySelector('div'),36    };37  });38  await page.evaluate(() => {39    window.customQueryHandlers = undefined;40  });41  await browser.close();42})();43const puppeteer = require('puppeteer');44(async () => {45  const browser = await puppeteer.launch();46  const page = await browser.newPage();47  await page.evaluate(() => {48    window.customQueryHandlers = {49      custom: () => document.querySelector('div'),50    };51  });52  await page.evaluate(() => {53    window.customQueryHandlers = null;54  });55  await browser.close();56})();57const puppeteer = require('puppeteer');58(async () => {59  const browser = await puppeteer.launch();60  const page = await browser.newPage();61  await page.evaluate(() =>

Using AI Code Generation

1const puppeteer = require('puppeteer');2(async () => {3    const browser = await puppeteer.launch();4    const page = await browser.newPage();5    await page.waitForSelector('input[title="Search"]');6    await page.type('input[title="Search"]', 'Puppeteer');7    await page.keyboard.press('Enter');8    await page.waitForNavigation();9    await page.waitForSelector('h3');10    await page.evaluate(() => {11        window.clearCustomQueryHandlers();12    });13    await page.waitForSelector('h3');14    await page.close();15    await browser.close();16})();

Using AI Code Generation

1const { PuppeteerCrawler } = require('apify');2const crawler = new PuppeteerCrawler({3    handlePageFunction: async () => {},4    handleFailedRequestFunction: async () => {},5    sessionPoolOptions: {6        sessionOptions: {7        },8    },9});10const { CheerioCrawler } = require('apify');11const crawler = new CheerioCrawler({12    handlePageFunction: async () => {},13    handleFailedRequestFunction: async () => {},14    sessionPoolOptions: {15        sessionOptions: {16        },17    },18});19const { PlaywrightCrawler } = require('apify');20const crawler = new PlaywrightCrawler({21    handlePageFunction: async () => {},22    handleFailedRequestFunction: async () => {},23    sessionPoolOptions: {24        sessionOptions: {25        },26    },27});28const { PuppeteerPool } = require('apify');29const pool = new PuppeteerPool({30    launchPuppeteerFunction: async () => {},31});

Using AI Code Generation

1const puppeteer = require('puppeteer');2const browser = await puppeteer.launch({headless:false});3const page = await browser.newPage();4await page.setRequestInterception(true);5await page.on('request', (req) => {6    if (req.resourceType() == 'image') {7        req.abort();8    }9    else {10        req.continue();11    }12});13await page.screenshot({path: 'google.png'});14await page.setRequestInterception(false);15await page.screenshot({path: 'google.png'});16await browser.close();

Using AI Code Generation

1page.setCustomQueryHandler(name, queryHandler)2await page.setCustomQueryHandler('newQueryHandler', (name, options) => {3});4page.customQueryHandler(name, options)5await page.customQueryHandler('newQueryHandler', {6});7page.clearCustomQueryHandlers(name)8await page.clearCustomQueryHandlers('newQueryHandler');

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.