Best JavaScript code snippet using playwright-internal
network.js
Source:network.js
...179 bodySize() {180 var _this$postDataBuffer;181 return ((_this$postDataBuffer = this.postDataBuffer()) === null || _this$postDataBuffer === void 0 ? void 0 : _this$postDataBuffer.length) || 0;182 }183 async requestHeadersSize() {184 let headersSize = 4; // 4 = 2 spaces + 2 line breaks (GET /path \r\n)185 headersSize += this.method().length;186 headersSize += new URL(this.url()).pathname.length;187 headersSize += 8; // httpVersion188 const headers = this.rawRequestHeadersPromise() ? await this.rawRequestHeadersPromise() : this._headers;189 for (const header of headers) headersSize += header.name.length + header.value.length + 4; // 4 = ': ' + '\r\n'190 return headersSize;191 }192}193exports.Request = Request;194class Route extends _instrumentation.SdkObject {195 constructor(request, delegate) {196 super(request.frame(), 'route');197 this._request = void 0;198 this._delegate = void 0;199 this._handled = false;200 this._request = request;201 this._delegate = delegate;202 }203 request() {204 return this._request;205 }206 async abort(errorCode = 'failed') {207 (0, _utils.assert)(!this._handled, 'Route is already handled!');208 this._handled = true;209 await this._delegate.abort(errorCode);210 }211 async fulfill(overrides) {212 (0, _utils.assert)(!this._handled, 'Route is already handled!');213 this._handled = true;214 let body = overrides.body;215 let isBase64 = overrides.isBase64 || false;216 if (body === undefined) {217 if (overrides.fetchResponseUid) {218 const context = this._request.frame()._page._browserContext;219 const buffer = context.fetchRequest.fetchResponses.get(overrides.fetchResponseUid) || _fetch.APIRequestContext.findResponseBody(overrides.fetchResponseUid);220 (0, _utils.assert)(buffer, 'Fetch response has been disposed');221 body = buffer.toString('base64');222 isBase64 = true;223 } else {224 body = '';225 isBase64 = false;226 }227 }228 await this._delegate.fulfill({229 status: overrides.status || 200,230 headers: overrides.headers || [],231 body,232 isBase64233 });234 }235 async continue(overrides = {}) {236 (0, _utils.assert)(!this._handled, 'Route is already handled!');237 if (overrides.url) {238 const newUrl = new URL(overrides.url);239 const oldUrl = new URL(this._request.url());240 if (oldUrl.protocol !== newUrl.protocol) throw new Error('New URL must have same protocol as overridden URL');241 }242 await this._delegate.continue(this._request, overrides);243 }244}245exports.Route = Route;246class Response extends _instrumentation.SdkObject {247 constructor(request, status, statusText, headers, timing, getResponseBodyCallback, httpVersion) {248 super(request.frame(), 'response');249 this._request = void 0;250 this._contentPromise = null;251 this._finishedPromise = new _async.ManualPromise();252 this._status = void 0;253 this._statusText = void 0;254 this._url = void 0;255 this._headers = void 0;256 this._headersMap = new Map();257 this._getResponseBodyCallback = void 0;258 this._timing = void 0;259 this._serverAddrPromise = new _async.ManualPromise();260 this._securityDetailsPromise = new _async.ManualPromise();261 this._rawResponseHeadersPromise = void 0;262 this._httpVersion = void 0;263 this._request = request;264 this._timing = timing;265 this._status = status;266 this._statusText = statusText;267 this._url = request.url();268 this._headers = headers;269 for (const {270 name,271 value272 } of this._headers) this._headersMap.set(name.toLowerCase(), value);273 this._getResponseBodyCallback = getResponseBodyCallback;274 this._request._setResponse(this);275 this._httpVersion = httpVersion;276 }277 _serverAddrFinished(addr) {278 this._serverAddrPromise.resolve(addr);279 }280 _securityDetailsFinished(securityDetails) {281 this._securityDetailsPromise.resolve(securityDetails);282 }283 _requestFinished(responseEndTiming) {284 this._request._responseEndTiming = Math.max(responseEndTiming, this._timing.responseStart);285 this._finishedPromise.resolve();286 }287 _setHttpVersion(httpVersion) {288 this._httpVersion = httpVersion;289 }290 url() {291 return this._url;292 }293 status() {294 return this._status;295 }296 statusText() {297 return this._statusText;298 }299 headers() {300 return this._headers;301 }302 headerValue(name) {303 return this._headersMap.get(name);304 }305 async rawResponseHeaders() {306 return this._rawResponseHeadersPromise || Promise.resolve(this._headers);307 }308 setWillReceiveExtraHeaders() {309 this._request.setWillReceiveExtraHeaders();310 this._rawResponseHeadersPromise = new _async.ManualPromise();311 }312 setRawResponseHeaders(headers) {313 if (!this._rawResponseHeadersPromise) this._rawResponseHeadersPromise = new _async.ManualPromise();314 this._rawResponseHeadersPromise.resolve(headers);315 }316 timing() {317 return this._timing;318 }319 async serverAddr() {320 return (await this._serverAddrPromise) || null;321 }322 async securityDetails() {323 return (await this._securityDetailsPromise) || null;324 }325 body() {326 if (!this._contentPromise) {327 this._contentPromise = this._finishedPromise.then(async () => {328 if (this._status >= 300 && this._status <= 399) throw new Error('Response body is unavailable for redirect responses');329 return this._getResponseBodyCallback();330 });331 }332 return this._contentPromise;333 }334 request() {335 return this._request;336 }337 frame() {338 return this._request.frame();339 }340 httpVersion() {341 if (!this._httpVersion) return 'HTTP/1.1';342 if (this._httpVersion === 'http/1.1') return 'HTTP/1.1';343 if (this._httpVersion === 'h2') return 'HTTP/2.0';344 return this._httpVersion;345 }346 async _responseHeadersSize() {347 if (this._request.responseSize.responseHeadersSize) return this._request.responseSize.responseHeadersSize;348 let headersSize = 4; // 4 = 2 spaces + 2 line breaks (HTTP/1.1 200 Ok\r\n)349 headersSize += 8; // httpVersion;350 headersSize += 3; // statusCode;351 headersSize += this.statusText().length;352 const headers = await this._bestEffortResponseHeaders();353 for (const header of headers) headersSize += header.name.length + header.value.length + 4; // 4 = ': ' + '\r\n'354 headersSize += 2; // '\r\n'355 return headersSize;356 }357 async _bestEffortResponseHeaders() {358 return this._rawResponseHeadersPromise ? await this._rawResponseHeadersPromise : this._headers;359 }360 async sizes() {361 await this._finishedPromise;362 const requestHeadersSize = await this._request.requestHeadersSize();363 const responseHeadersSize = await this._responseHeadersSize();364 let {365 encodedBodySize366 } = this._request.responseSize;367 if (!encodedBodySize) {368 var _headers$find;369 const headers = await this._bestEffortResponseHeaders();370 const contentLength = (_headers$find = headers.find(h => h.name.toLowerCase() === 'content-length')) === null || _headers$find === void 0 ? void 0 : _headers$find.value;371 encodedBodySize = contentLength ? +contentLength : 0;372 }373 return {374 requestBodySize: this._request.bodySize(),375 requestHeadersSize,376 responseBodySize: encodedBodySize,...
ResponseHistorySaver.js
Source:ResponseHistorySaver.js
1/**2@license3Copyright 2018 The Advanced REST client authors <arc@mulesoft.com>4Licensed under the Apache License, Version 2.0 (the "License"); you may not5use this file except in compliance with the License. You may obtain a copy of6the License at7http://www.apache.org/licenses/LICENSE-2.08Unless required by applicable law or agreed to in writing, software9distributed under the License is distributed on an "AS IS" BASIS, WITHOUT10WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the11License for the specific language governing permissions and limitations under12the License.13*/14import '@advanced-rest-client/uuid-generator/uuid-generator.js';15import 'pouchdb/dist/pouchdb.js';16/**17 * An element that saves requests history in a datastore.18 *19 * This element supports Advanced REST Client project.20 *21 * It handles the `api-response` event asynchronously and updates both22 * requests history and history data data store.23 *24 * The requests history keeps a daily record of requests made by the application.25 * It keeps record of rhe request data that can be restored lated by the26 * application.27 *28 * The history data keeps record of every request made by the application.29 * It can be used to analyse performance of an API endpoint.30 *31 * ## Data model32 *33 * ### request data34 *35 * Note that payload is always string even if the response body was different type.36 *37 * Property | Type | Description38 * ----------------|-------------|-------------39 * `_id` | `String` | PouchDB database key.40 * `timings` | `Object` | Valid HAR 1.2 timings object.41 * `totalTime` | `Number` | Number of milliseconds that took to perform the full request.42 * `created` | `Number` | Timestamp of the entry43 * `request` | `Object` | A request details object (see below).44 * `request.headers` | `String` | HTTP headers string sent to the server.45 * `request.payload` | `String` | HTTP message string set to the server.46 * `request.url` | `String` | Request URL47 * `request.method` | `String` | HTTP method of the request48 * `response` | `Object` | Response details object49 * `response.statusCode` | `Number` | A status code of the response.50 * `response.statusText` | `String` | Status text message. Can be empty or undefined.51 * `response.headers` | `String` | HTTP headers string of the response.52 * `response.payload` | `String` | Response body string.53 * `stats` | `Object` | Request and response basic statistics54 * `stats.request` | `Object` | Request basic statistics55 * `stats.request.headersSize` | `Number` | Request headers size in bytes56 * `stats.request.payloadSize` | `Number` | Request payload size in bytes57 * `stats.response` | `Object` | Response basic statistics58 * `stats.response.headersSize` | `Number` | Response headers size in bytes59 * `stats.response.payloadSize` | `Number` | Response payload size in bytes60 *61 * @customElement62 * @demo demo/index.html63 * @memberof LogicElements64 */65export class ResponseHistorySaver extends HTMLElement {66 /**67 * @return {PouchDB} A PouchDB instance for the history-data store.68 */69 get _dbData() {70 /* global PouchDB */71 return new PouchDB('history-data');72 }73 get uuid() {74 /* istanbul ignore else */75 if (!this.__uuid) {76 this.__uuid = document.createElement('uuid-generator');77 }78 return this.__uuid;79 }80 constructor() {81 super();82 this._afterRequestHandler = this._afterRequestHandler.bind(this);83 }84 connectedCallback() {85 window.addEventListener('api-response', this._afterRequestHandler);86 if (!this.hasAttribute('aria-hidden')) {87 this.setAttribute('aria-hidden', 'true');88 }89 }90 disconnectedCallback() {91 window.removeEventListener('api-response', this._afterRequestHandler);92 this.__uuid = null;93 }94 /**95 * Handler for the `api-response` event96 *97 * @param {CustomEvent} e98 */99 _afterRequestHandler(e) {100 const { isError, response, request, timing } = e.detail;101 /* istanbul ignore if */102 if (isError) {103 return;104 }105 // Async so the response can be rendered to the user faster.106 setTimeout(() => this.saveHistory(request, response, timing));107 }108 /**109 * Saves request and response data in history.110 *111 * @param {Object} request ARC request object112 * @param {Object} response ARC response object113 * @param {Object} timings Request timings as HAR 1.2 timings object114 * @return {Promise}115 */116 async saveHistory(request, response, timings) {117 try {118 await this._saveHistoryData(request, response, timings);119 return await this._updateHistory(request, timings);120 } catch (cause) {121 this._handleException(cause);122 }123 }124 /**125 * Saves history data for history analysis in the data store.126 *127 * @param {Request} request The request object containg information about128 * the request129 * @param {Response} response The response object for the response130 * @param {?Object} eventTimings The timings object as descrived in HAR 1.2 spec. Optional.131 * @return {Promise} A promise resolved when data were inserted to the132 * datastore.133 */134 async _saveHistoryData(request, response, eventTimings) {135 const doc = this._createHistoryDataModel(request, response, eventTimings);136 const db = this._dbData;137 try {138 return await db.put(doc);139 } catch (e) {140 this._handleException(e);141 }142 }143 // Creates a data store model for `_saveHistoryData`144 _createHistoryDataModel(request, response, eventTimings) {145 const requestHeaders = request.headers;146 const responseHeaders = response.headers;147 const timings = this._computeTimings(eventTimings);148 const totalTime = this._computeTotalTime(timings);149 const requestPayloadSize = this._computePayloadSize(request.payload);150 const responsePayloadSize = this._computePayloadSize(response.payload);151 const requestHeadersSize = this._calculateBytes(requestHeaders);152 const responseHeadersSize = this._calculateBytes(responseHeaders);153 const requestStartTime = this._computeStartTime(eventTimings);154 const url = this._computeHistoryStoreUrl(request.url);155 const id = this._computeHistoryDataId(url, request.method);156 const requestPayload = this._computePayloadString(request.payload);157 const responsePayload = this._computePayloadString(response.payload);158 const doc = {159 _id: id,160 timings: timings,161 totalTime: totalTime,162 created: requestStartTime,163 request: {164 headers: requestHeaders,165 payload: requestPayload,166 url: request.url,167 method: request.method,168 },169 response: {170 statusCode: response.status,171 statusText: response.statusText,172 headers: responseHeaders,173 payload: responsePayload174 },175 stats: {176 request: {177 headersSize: requestHeadersSize,178 payloadSize: requestPayloadSize179 },180 response: {181 headersSize: responseHeadersSize,182 payloadSize: responsePayloadSize183 }184 }185 };186 return doc;187 }188 /**189 * Updates the requests history data store.190 * If the request for given URL and method has been already performed this191 * day then the record in the datastore is updated with new data.192 * Otherwise a new record is created.193 *194 * @param {Request} request The request object.195 * @param {?Object} eventTimings A HAR 1.2 timings object from the response196 * event.197 * @return {Promise} A promise that is resolved then the history object has198 * been updated in the data store.199 */200 _updateHistory(request, eventTimings) {201 const updated = this._computeStartTime(eventTimings);202 const d = new Date();203 d.setHours(0, 0, 0, 0);204 let id = d.getTime();205 id += '/' + encodeURIComponent(request.url);206 id += '/' + request.method;207 const doc = {208 _id: id,209 updated,210 created: updated,211 headers: request.headers,212 method: request.method,213 payload: request.payload,214 url: request.url215 };216 if (request.auth && request.authType) {217 doc.auth = request.auth;218 doc.authType = request.authType;219 }220 if (request.config) {221 doc.config = request.config;222 }223 if (request.requestActions) {224 doc.requestActions = request.requestActions;225 }226 if (request.responseActions && request.responseActions.length) {227 doc.responseActions = request.responseActions;228 }229 if (request._response || request.response) {230 doc.response = this._prepareResponse(request._response || request.response);231 }232 if (request._responseMeta || request.responseMeta) {233 doc.responseMeta = request._responseMeta || request.responseMeta;234 }235 if (request._state || request.state) {236 doc.state = request._state || request.state;237 }238 if (typeof request._isErrorResponse === 'boolean') {239 doc.isErrorResponse = request._isErrorResponse;240 } else if (typeof request.isErrorResponse === 'boolean') {241 doc.isErrorResponse = request.isErrorResponse;242 }243 const e = new CustomEvent('save-history', {244 bubbles: true,245 composed: true,246 cancelable: true,247 detail: {248 request: doc249 }250 });251 this.dispatchEvent(e);252 if (!e.defaultPrevented) {253 return Promise.reject(new Error('request model not found'));254 }255 return e.detail.result;256 }257 /**258 * Replaces buffer with base64 string in response's payload259 * @param {Object} response ARC response object260 * @return {Object} Copy of the object261 */262 _prepareResponse(response) {263 const { payload } = response;264 if (!payload) {265 return response;266 }267 response = Object.assign({}, response);268 if (payload.asciiSlice) {269 // node's buffer270 response.payload = payload.toString('base64');271 }272 if (payload.byteLength) {273 // Array buffer274 response.payload = btoa(String.fromCharCode(...payload));275 }276 return response;277 }278 /**279 * Computes a valid timings object as descrived in HAR 1.2 spec.280 *281 * @param {?Object} eventTimings A timings object passed by the response282 * event283 * @return {Object} A valid HAR 1.2 timings object.284 */285 _computeTimings(eventTimings) {286 eventTimings = eventTimings || {};287 const timings = {288 connect: eventTimings.connect || -1,289 receive: eventTimings.receive || -1,290 send: eventTimings.send || -1,291 ssl: eventTimings.ssl || -1,292 wait: eventTimings.wait || -1293 };294 return timings;295 }296 /**297 * Computes a timestamp of the request start time.298 * If timings object provided with the response event contains `startTime`299 * property it will compute timestamp from it. Otherwise it will use300 * current time.301 *302 * @param {?Object} eventTimings Timings object from the response event.303 * @return {Number} Timestamp of the request start.304 */305 _computeStartTime(eventTimings) {306 eventTimings = eventTimings || {};307 return eventTimings.startTime ?308 new Date(eventTimings.startTime).getTime() :309 Date.now();310 }311 /**312 * Computes total time of the request from the timings object.313 *314 * @param {Object} timings A timings object as described in HAR 1.2 spec.315 * @return {Number} Sum of times in the `timings` object. The `-1` values316 * doeasn't adds. It returns `-1` if all values are -1.317 */318 _computeTotalTime(timings) {319 const values = Object.keys(timings).map((key) => timings[key]);320 let total = values.reduce((sum, value) => {321 if (value > -1) {322 return sum + value;323 }324 return sum;325 }, 0);326 if (total === 0) {327 total = -1;328 }329 return total;330 }331 /**332 * Produces valid URL to be used in the history-data store.333 * The URL is stripped from query parameters and hash.334 *335 * @param {String} url A URL to process336 * @return {String}337 */338 _computeHistoryStoreUrl(url) {339 try {340 url = new URL(url);341 url.search = '';342 url.hash = '';343 url = url.toString();344 } catch (e) {345 // ..346 }347 if (url) {348 let i = url.indexOf('?');349 if (i !== -1) {350 url = url.substr(0, i);351 }352 i = url.indexOf('#');353 if (i !== -1) {354 url = url.substr(0, i);355 }356 }357 return url;358 }359 /**360 * Computes size of the payload.361 *362 * @param {ArrayBuffer|Blob|String} payload The payload363 * @return {Number} Size of the payload364 */365 _computePayloadSize(payload) {366 if (!payload) {367 return 0;368 }369 if (payload instanceof ArrayBuffer) {370 return payload.byteLength;371 } else if (payload instanceof Blob) {372 return payload.size;373 } else {374 return this._calculateBytes(payload);375 }376 }377 /**378 * Calculates size of the string379 * @param {String} str A string to compute size from.380 * @return {Number} Size of the string.381 */382 _calculateBytes(str) {383 if (!str || !str.length || typeof str !== 'string') {384 return 0;385 }386 let s = str.length;387 for (let i = str.length - 1; i >= 0; i--) {388 const code = str.charCodeAt(i);389 if (code > 0x7f && code <= 0x7ff) {390 s++;391 } else if (code > 0x7ff && code <= 0xffff) {392 /* istanbul ignore next */393 s += 2;394 }395 /* istanbul ignore if */396 if (code >= 0xDC00 && code <= 0xDFFF) {397 i--; // trail surrogate398 }399 }400 return s;401 }402 /**403 * Computes an ID for the `history-data` datas tore.404 *405 * @param {String} url Request URL stripped from query parameters and406 * the hash.407 * @param {String} method HTTP method name.408 * @return {String} Generated unique for this request data store ID. It uses409 * UUID generator to add some random data to the ID except for the410 * URL and method.411 */412 _computeHistoryDataId(url, method) {413 const uuid = this.uuid;414 const id = encodeURIComponent(url) + '/' + method + '/' +415 uuid.generate();416 return id;417 }418 /**419 * Computes a payload message as a string.420 *421 * @param {any} input Request or response payload422 * @return {String}423 */424 _computePayloadString(input) {425 const result = '';426 if (!input) {427 return result;428 }429 if (typeof input === 'string') {430 return input;431 }432 if (input instanceof ArrayBuffer) {433 return this._arrayBufferToString(input);434 }435 if (input instanceof Uint8Array) {436 return input.toString();437 }438 return result;439 }440 /**441 * Convert ArrayBuffer to readable form442 * @param {ArrayBuffer} buffer443 * @return {String} Converted string444 */445 _arrayBufferToString(buffer) {446 if (buffer.buffer) {447 const b = buffer.slice(0);448 buffer = b.buffer;449 }450 const decoder = new TextDecoder();451 try {452 return decoder.decode(buffer);453 } catch (_) {454 return '';455 }456 }457 /**458 * Handles exceptions to log message ad throws the same exception459 *460 * @param {Error} e461 */462 _handleException(e) {463 let message;464 if (e instanceof Error) {465 message = e.message;466 } else {467 message = JSON.stringify(e);468 }469 this.dispatchEvent(new CustomEvent('send-analytics', {470 composed: true,471 bubbles: true,472 detail: {473 type: 'exception',474 description: message,475 fatal: false476 }477 }));478 throw new Error(e.message || e);479 }...
HAR_Headless_Parser.js
Source:HAR_Headless_Parser.js
1/********************************************2Dependencies3********************************************/4const fs = require('fs'); // file system5const chc = require('chrome-har-capturer'); // capture HAR libraru6const argv = require('minimist')(process.argv.slice(2)); // used for easy param parsing7const readlineSync = require('readline-sync'); // reads input synchonously8const mysql = require('mysql'); // offical mysql library9const async = require('async');10/********************************************11Globals12********************************************/13const COMPUTER_TYPE = 1; //TODO 0 == H2, 1 == http114const CONNECTION_PATH = 0; // 0 || 1 == nginx, 2 == apache W/SP, 3 == apache Wo/SP15const EXTRA_CONFIG = 2; // 1 = ethernet, 2 = Wifi16var DB_NAME, DB_HOST, DB_USER, DB_PASS; // used to log into database17var INPUT_FILE; // where to grab the websites18var PORT = 9222; // for chrome-har-captuer19var VERBOSE = false; // when you want your terminal spammed20var DEBUG = false; // used for turning on debug output21var WEBSITE_LIST; // array of websites to scan22var PARSE_LOOP_COUNT = 0; // used to track recursive loop23/********************************************24Param Checking and Validation25********************************************/26// -h, --help27if (argv.h || argv.help) {28 print_options();29 process.exit(1);30}31// --dbhost32if (argv.dbhost) {33 DB_HOST = argv.dbhost;34} else {35 DB_HOST = readlineSync.question("Enter IP of database: (127.0.0.1) ");36 if (DB_HOST == false) { DB_HOST = "127.0.0.1"; } // default37}38// --dbport39if (argv.dbport) {40 DB_PORT = argv.dbport;41} else {42 DB_PORT = readlineSync.question("Enter port of database: (3306) ");43 if (DB_PORT == false) { DB_PORT = "3306"; } // default44}45// --dbuser46if (argv.dbuser) {47 DB_USER = argv.dbuser;48} else {49 DB_USER = readlineSync.question("Enter username of database: (none) ");50}51// --dbpass52if (argv.dbpass) {53 DB_PASS = argv.dbpass;54} else {55 // TODO make password not show when typing56 DB_PASS = readlineSync.question("Enter password of database: (none) ");57}58// --dbname59if (argv.dbname) {60 DB_NAME = argv.dbname;61} else {62 while (!DB_NAME) {63 DB_NAME = readlineSync.question("Enter database name: ");64 }65}66// -i, --input67if (argv.i || argv.input) {68 INPUT_FILE = argv.input || argv.i;69 if (!fs.existsSync(INPUT_FILE)) {70 console.log("Input file does not exist!");71 while (!INPUT_FILE) {72 INPUT_FILE = readlineSync.question("Enter input file: ");73 if (!fs.existsSync(INPUT_FILE)) {74 console.log("File does not exist!");75 }76 }77 }78} else {79 while (!INPUT_FILE) {80 INPUT_FILE = readlineSync.question("Enter input file: ");81 if (!fs.existsSync(INPUT_FILE)) {82 console.log("File does not exist!");83 }84 }85}86// parse file into array from line-by-line87WEBSITE_LIST = fs.readFileSync(INPUT_FILE).toString().split("\n");88// checks for empty lines89for (let i = 0; i < WEBSITE_LIST.length; ) {90 if (WEBSITE_LIST[i] == "" | WEBSITE_LIST[i] == false) {91 WEBSITE_LIST.splice(i,1);92 } else {93 i++; // if spliced the i value is already the next element94 }95}96// -p, --port97if (argv.p || argv.port) { PORT = argv.port || argv.p; }98// -v, --verbose99if (argv.v || argv.verbose) { VERBOSE = true; }100// --debug101if (argv.debug) { DEBUG = true; }102/********************************************103Main Parsing and Database Inserting104 ********************************************/105// Not using pooling due to tracking each insert over performance since no real time transactions needed106// More setting can be added but ignored for the general case107var connection = mysql.createConnection({108 host : DB_HOST,109 user : DB_USER,110 password : DB_PASS,111 database : DB_NAME,112 port : DB_PORT,113 dateStrings : 'date' // needed to allow javascript dates, MySQL will be forced to cast it114});115if (DEBUG) { console.log("DB_HOST: ",DB_HOST,"\nDB_USER: ",DB_USER,"\nDB_PASS: ",DB_PASS,"\nDB_NAME: ",DB_NAME,"\nDB_PORT: ",DB_PORT,"\n");}116connection.connect( (err) => {117 if (err) {118 console.error('ERROR: connecting to database: ' + err.stack);119 process.exit(1);120 }121 if (VERBOSE) { console.log("Connected to database as threadId", connection.threadId, "\n"); }122 parse_loop();123});124/*125 * This is the main parsing loop that recursively runs126 * Currently best approach to synch all async functions127 */128function parse_loop() {129 // called when no more sites left130 if (PARSE_LOOP_COUNT >= WEBSITE_LIST.length) {131 cleanup();132 return;133 }134 // loads site and waits for event135 HAR_LOAD = chc.load(WEBSITE_LIST[PARSE_LOOP_COUNT], {"port": PORT});136 HAR_LOAD.on('connect', function () {137 if (VERBOSE) { console.log(WEBSITE_LIST[PARSE_LOOP_COUNT] + ' connected to Chrome\n'); }138 });139 // TODO, have it retry or something better then skip140 HAR_LOAD.on('error', (err) => {141 console.error('Cannot connect to Chrome for' + WEBSITE_LIST[PARSE_LOOP_COUNT] + ' due to : ' + err);142 console.error("TODO: Catch HAR_LOAD error");143 process.exit(1);144 });145 // takes har and inserts it146 HAR_LOAD.on('end', (har) => {147 // gets website obj data and parses it out for use in query insert148 // assumes the url contains name of html file at end WITHOUT .html149 var temp = WEBSITE_LIST[PARSE_LOOP_COUNT].substring(0, WEBSITE_LIST[PARSE_LOOP_COUNT].length - 1); //prevents if site ends with '/'150 var website_obj = temp.substring(temp.lastIndexOf("/")+1) // gets just the last part of path returns ex: W_1_2_a151 var obj_type = website_obj.split("_")[0];152 var obj_size = website_obj.split("_")[1];153 var obj_count = website_obj.split("_")[2];154 var obj_structure = website_obj.split("_")[3]; // TODO: add check for .html on url155 var domain = har.log.pages[0].title; // keep variable as we use is multiple times below156 var WebsiteID; // gets called from website_query and used as foreign key for entry insert query157 /** WEBSITE QUERY **/158 var website_query = connection.query('INSERT INTO ' + DB_NAME + '.Website SET ' +159 'Domain = ?, NumberOfFiles = ?, StartedDateTime = ?, OnContentLoad = ?, OnLoad = ?, ' +160 'ObjectType = ?, Size = ?, Count = ?, Structure = ?',161 [162 domain, // domain,163 har.log.entries.length, // NumberOfFiles164 new Date(har.log.pages[0].startedDateTime), // StartedDateTime165 har.log.pages[0].pageTimings.onContentLoad, // OnContentLoad166 har.log.pages[0].pageTimings.onLoad, // OnLoad167 obj_type, // ObjectType168 obj_size, // Size169 obj_count, // Count170 obj_structure // Structure171 ],172 (error, results, fields) => { // returns on website query insert173 if (error) {174 console.error("website_query error: ", error);175 console.log("TODO: Catch query error");176 process.exit(1);177 }178 WebsiteId = results.insertId;179 // loops through each asynch call in a synch fashion180 async.forEachSeries(har.log.entries,181 function(entry, callback) { // each entry table query182 // loops through all headers and will be null if not found as headers are optional183 var header_req_cache_control = findHeader(entry.request.headers, "cache-control");184 var header_req_date = findHeader(entry.request.headers, "date");185 var header_req_user_agent = findHeader(entry.request.headers, "user-agent");186 var header_res_date = findHeader(entry.response.headers, "date");187 var header_res_last_modified = findHeader(entry.response.headers, "last-modified");188 var header_res_server = findHeader(entry.response.headers, "server");189 var header_res_content_length = findHeader(entry.response.headers, "content-length");190 var entry_query = connection.query('INSERT INTO ' + DB_NAME + '.Entries SET WebsiteId = ?, ' +191 'Domain = ?, StartedDateTime = ?, TotalTime = ?, RequestCacheControl = ?, RequestDate = ?, RequestUserAgent = ?, '+192 'RequestUrl = ?, RequestHeadersSize = ?, RequestBodySize = ?, ResponseDate = ?, ResponseLastModified = ?, '+193 'ResponseServer = ?, ResponseContentLength = ?, ResponseStatus = ?, ResponseHeadersSize = ?, ResponseBodySize = ?, '+194 'ResponseHttpVersion = ?, ResponseTransferSize = ?, Blocked = ?, DNS = ?, Connect = ?, Send = ?, Wait = ?, '+195 'Receive = ?, SSLTime = ?, ComputerType = ?, ConnectionPath = ?, ExtraConfig = ?',196 [197 WebsiteId, // WebsiteId198 domain, // Domain199 new Date(entry.startedDateTime), // StartedDateTime200 entry.time, // TotalTime201 header_req_cache_control, // RequestCacheControl202 new Date(header_req_date), // RequestDate203 header_req_user_agent, // RequestUserAgent204 entry.request.url, // RequestUrl205 entry.request.headersSize, // RequestHeadersSize206 entry.request.bodySize, // RequestBodySize207 new Date(header_res_date), // ResponseDate208 new Date(header_res_last_modified), // ResponseLastModified209 header_res_server, // ResponseServer210 header_res_content_length, // ResponseContentLength211 entry.response.status, // ResponseStatus212 entry.response.headersSize, // ResponseHeadersSize213 entry.response.bodySize, // ResponseBodySize214 entry.response.httpVersion, //ResponseHttpVersion215 entry.response._transferSize, // ResponseTransferSize216 entry.timings.blocked, // Blocked217 entry.timings.dns, // DNS218 entry.timings.connect, // Connect219 entry.timings.send, // Send220 entry.timings.wait, // Wait221 entry.timings.receive, // Receive222 entry.timings.ssl, // SSLTime223 COMPUTER_TYPE, // ComputerType224 CONNECTION_PATH, // ConnectionPath225 EXTRA_CONFIG // ExtraConfig226 ],227 function(error, results) {228 // call with each query229 if (error) {230 console.error("entry_query error: ", error);231 console.log("TODO: Catch query error");232 process.exit(1);233 }234 if (VERBOSE) { console.log("Datbase insert for ", entry.request.url, "\n"); }235 callback(); // used to call next async entry236 }); // connection.query237 if (DEBUG) console.log(entry_query.sql, "\n");238 }, function(error) { // called when all entries are done239 if (error) {240 console.error("async series error: ", error);241 console.log("TODO: Catch async error");242 process.exit(1);243 }244 console.log("Parse loop ", PARSE_LOOP_COUNT, "of ", WEBSITE_LIST.length-1, " complete"); // only non-verbose printout245 PARSE_LOOP_COUNT++;246// parse_loop(); // recursion call247 setTimeout(function(){ parse_loop(); }, 300); //TODO, used on slow computers248 }); // forEachOfSeries249 }); // website_query250 if (DEBUG) { console.log(website_query.sql, "\n") };251 }); // HAR_LOAD.on(end)252} // parse_loop253function cleanup() {254 connection.end(); // close SQL connection for good practice255 process.exit(1); // remove only if need to continue after cleanup256}257/*258 * custom array.find for headers, really just finds name key that matchs and the value in it259 * header format: { name : "header_key", value : "header_value" }260 */261function findHeader(headers, name) {262 for (let i = 0; i < headers.length; i++) {263 if (headers[i].name == name) {264 return headers[i].value265 }266 }267 return null; // null if not found for SQL database as undefinded is not SQL type268}269/*270* Used to print out options supported271*/272function print_options(){273 console.log("Usage: node HAR_Headless_Parser [options]...\n\n"+274 "Options:\n\n"+275 " -h, --help Output usage information\n"+276 " --dbhost <IP> IP address for database [Will prompt otherwise] [Default: 127.0.0.1]\n"+277 " --dbport <port> Port on machine to access database [Will prompt otherwise] [Default: 3306]\n"+278 " --dbuser <user> User of database [Will prompt otherwise]\n"+279 " --dbpass <pass> Password of database [Will prompt otherwise]\n"+280 " --dbname <name> Name of database to store data [Will prompt otherwise]\n"+281 " -i, --input <file> File of website input list in line-by-line fashion [Will prompt otherwise]\n"+282 " -p, --port <port> Remote Debugging Protocol port [Default: 9222]\n"+283 " -v, --verbose Enable verbose output on stdout\n"284 );...
HistoryDataModel.js
Source:HistoryDataModel.js
1/* eslint-disable no-plusplus */2/* eslint-disable class-methods-use-this */3/**4@license5Copyright 2017 The Advanced REST client authors <arc@mulesoft.com>6Licensed under the Apache License, Version 2.0 (the "License"); you may not7use this file except in compliance with the License. You may obtain a copy of8the License at9http://www.apache.org/licenses/LICENSE-2.010Unless required by applicable law or agreed to in writing, software11distributed under the License is distributed on an "AS IS" BASIS, WITHOUT12WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the13License for the specific language governing permissions and limitations under14the License.15*/16import { v4 } from '@advanced-rest-client/uuid';17import { TransportEventTypes, ArcModelEvents } from '@advanced-rest-client/events';18import { BodyProcessor } from '@advanced-rest-client/libs';19import { ArcBaseModel } from './ArcBaseModel.js';20import { normalizeRequest } from './Utils.js';21import { computePayloadSize, calculateBytes } from './lib/DataSize.js';22/** @typedef {import('@advanced-rest-client/events').ApiResponseEvent} ApiResponseEvent */23/** @typedef {import('@advanced-rest-client/events').ArcRequest.TransportRequest} TransportRequest */24/** @typedef {import('@advanced-rest-client/events').ArcRequest.ArcBaseRequest} ArcBaseRequest */25/** @typedef {import('@advanced-rest-client/events').ArcRequest.ARCHistoryRequest} ARCHistoryRequest */26/** @typedef {import('@advanced-rest-client/events').ArcResponse.Response} Response */27/** @typedef {import('@advanced-rest-client/events').ArcResponse.ErrorResponse} ErrorResponse */28/** @typedef {import('@advanced-rest-client/events').ArcResponse.RequestTime} RequestTime */29/** @typedef {import('@advanced-rest-client/events').ArcResponse.TransformedPayload} TransformedPayload */30/** @typedef {import('@advanced-rest-client/events').HistoryData.HistoryData} HistoryData */31export const responseHandler = Symbol('responseHandler');32export const saveHistoryData = Symbol('saveHistoryData');33export const updateHistory = Symbol('saveHistoryData');34export const createHistoryDataModel = Symbol('createHistoryDataModel');35export const createEmptyTimings = Symbol('createEmptyTimings');36export const computeHistoryStoreUrl = Symbol('computeHistoryStoreUrl');37export const computeHistoryDataId = Symbol('computeHistoryDataId');38export const prepareResponseBody = Symbol('prepareResponseBody');39export const computeTotalTime = Symbol('computeTotalTime');40/**41 * The model that stores requests history object (for the history menu and panel)42 * and the HAR-like object for each request made with ARC.43 */44export class HistoryDataModel extends ArcBaseModel {45 // static get properties() {46 // return {47 // /** 48 // * When set the element won't store history request as ARC history.49 // * It does not change the history data.50 // */51 // historyDisabled: { type: Boolean },52 // /** 53 // * When set the element won't store history data.54 // * It does not change the history request.55 // */56 // dataDisabled: { type: Boolean },57 // };58 // }59 constructor() {60 super('history-data');61 this.historyDisabled = false;62 this.dataDisabled = false;63 this[responseHandler] = this[responseHandler].bind(this);64 }65 /**66 * @param {EventTarget} node67 */68 listen(node) {69 super.listen(node);70 node.addEventListener(TransportEventTypes.response, this[responseHandler]);71 }72 /**73 * @param {EventTarget} node74 */75 unlisten(node) {76 super.unlisten(node);77 node.removeEventListener(TransportEventTypes.response, this[responseHandler]);78 }79 /**80 * Processes API response action.81 * @param {ApiResponseEvent} e82 */83 [responseHandler](e) {84 const { request, response, source } = e.detail;85 const typedError = /** @type ErrorResponse */ (response);86 if (typedError.error) {87 return;88 }89 const typedResponse = /** @type Response */ (response);90 // Async so the response can be rendered to the user faster.91 setTimeout(() => this.saveHistory(request, typedResponse, source));92 }93 /**94 * Saves request and response data in history.95 *96 * @param {TransportRequest} request The transport generated request object97 * @param {Response} response ARC response object98 * @param {ArcBaseRequest} source The source request object99 * @return {Promise<void>}100 */101 async saveHistory(request, response, source) {102 try {103 await this[saveHistoryData](request, response);104 await this[updateHistory](source, request, response);105 } catch (cause) {106 this._handleException(cause);107 }108 }109 /**110 * Saves historical request and response data in the data store.111 * This (for now) has no UI surface and is not technically processed. It is to be used112 * to analyze the data for API generation and also to show history analysis.113 * @param {TransportRequest} request The transport generated request object114 * @param {Response} response ARC response object115 * @return {Promise<void>}116 */117 async [saveHistoryData](request, response) {118 if (this.dataDisabled) {119 return;120 }121 const doc = await this[createHistoryDataModel](request, response);122 try {123 await this.db.put(doc);124 } catch (e) {125 this._handleException(e);126 }127 }128 /**129 * Creates an entry in the ARC request history130 * @param {ArcBaseRequest} source The source request from the editor131 * @param {TransportRequest} request The transport generated request object132 * @param {Response} response ARC response133 * @return {Promise<void>}134 */135 async [updateHistory](source, request, response) {136 if (this.historyDisabled) {137 return;138 }139 const responseCopy = { ...response };140 responseCopy.payload = this[prepareResponseBody](responseCopy.payload);141 const d = new Date();142 d.setHours(0, 0, 0, 0);143 const encoded = encodeURIComponent(source.url);144 const id = `${d.getTime()}/${encoded}/${source.method}`;145 const copy = { ...source, type: 'history', _id: id };146 const normalized = normalizeRequest(copy);147 const transportCopy = await BodyProcessor.payloadToString(normalizeRequest(request));148 const created = request.startTime;149 const updated = request.endTime;150 const doc = /** @type ARCHistoryRequest */ ({151 ...normalized,152 updated,153 created,154 response: responseCopy,155 transportRequest: transportCopy,156 });157 await ArcModelEvents.Request.store(this.eventsTarget, 'history', doc);158 }159 /**160 * @param {TransportRequest} request The transport generated request object161 * @param {Response} response ARC response object162 * @returns {Promise<HistoryData>} 163 */164 async [createHistoryDataModel](request, response) {165 const url = this[computeHistoryStoreUrl](request.url);166 const id = this[computeHistoryDataId](url, request.method);167 const timings = response.timings || this[createEmptyTimings]();168 const totalTime = this[computeTotalTime](timings);169 const requestPayloadSize = await computePayloadSize(request.payload);170 const responsePayloadSize = await computePayloadSize(response.payload);171 const requestHeadersSize = calculateBytes(request.headers);172 const responseHeadersSize = calculateBytes(response.headers);173 const requestCopy = await BodyProcessor.payloadToString(request);174 const responsePayload = this[prepareResponseBody](response.payload);175 const doc = /** @type HistoryData */ ({176 _id: id,177 timings,178 totalTime,179 created: request.startTime,180 request: {181 headers: request.headers,182 payload: requestCopy.payload,183 url: request.url,184 method: request.method,185 // @ts-ignore186 multipart: requestCopy.multipart,187 // @ts-ignore188 blob: requestCopy.blob,189 },190 response: {191 statusCode: response.status,192 statusText: response.statusText,193 headers: response.headers,194 payload: responsePayload,195 },196 stats: {197 request: {198 headersSize: requestHeadersSize,199 payloadSize: requestPayloadSize,200 },201 response: {202 headersSize: responseHeadersSize,203 payloadSize: responsePayloadSize,204 }205 }206 });207 return doc;208 }209 /**210 * @returns {RequestTime} Empty request timings in case they were missing.211 */212 [createEmptyTimings]() {213 return {214 blocked: -1,215 connect: -1,216 receive: -1,217 send: -1,218 ssl: -1,219 wait: -1,220 dns: -1,221 };222 }223 /**224 * Produces valid URL to be used in the history-data store.225 * The URL is stripped from query parameters and hash.226 *227 * @param {string} url A URL to process228 * @returns {string} Indexable history data id229 */230 [computeHistoryStoreUrl](url) {231 let value = url;232 try {233 const parser = new URL(url);234 parser.search = '';235 parser.hash = '';236 value = parser.toString();237 } catch (e) {238 // ..239 }240 if (value) {241 let i = value.indexOf('?');242 if (i !== -1) {243 value = value.substr(0, i);244 }245 i = value.indexOf('#');246 if (i !== -1) {247 value = value.substr(0, i);248 }249 }250 return value;251 }252 /**253 * Computes an id value for the `history-data` data store.254 *255 * @param {string} url The request URL with removed query parameters and the hash.256 * @param {string} method HTTP method name.257 * @return {string} Generated unique for this request data store ID. It uses258 * UUID generator to add some random data to the ID except for the259 * URL and method.260 */261 [computeHistoryDataId](url, method) {262 const uuid = v4();263 const encoded = encodeURIComponent(url);264 return `${encoded}/${method}/${uuid}`;265 }266 /**267 * Replaces buffer with base64 string in response's payload268 * @param {string|Buffer|ArrayBuffer|TransformedPayload} body The response body269 * @returns {string|TransformedPayload} Either the string (when input is a string) or270 * an object with the original type and data 271 */272 [prepareResponseBody](body) {273 if (!body) {274 return undefined;275 }276 const typedBuffer = /** @type Buffer */(body);277 /* istanbul ignore if */278 if (typeof typedBuffer.copy === 'function') {279 return {280 type: 'Buffer',281 data: [...typedBuffer],282 };283 }284 const typedArrayBuffer = /** @type ArrayBuffer */(body);285 if (typedArrayBuffer.byteLength) {286 const view = new Uint8Array(typedArrayBuffer);287 return {288 type: 'ArrayBuffer',289 data: Array.from(view),290 };291 }292 return String(body);293 }294 /**295 * Computes total time of the request from the timings object.296 *297 * @param {RequestTime} timings Rhe request timings.298 * @return {number} Sum of times in the `timings` object. The `-1` values are ignored.299 */300 [computeTotalTime](timings) {301 const values = Object.keys(timings).map((key) => timings[key]);302 let total = values.reduce((sum, value) => {303 if (value > -1) {304 return sum + value;305 }306 return sum;307 }, 0);308 if (total === 0) {309 total = -1;310 }311 return total;312 }...
HAREntry.js
Source:HAREntry.js
1/*2 * Copyright (C) 2011 Google Inc. All rights reserved.3 *4 * Redistribution and use in source and binary forms, with or without5 * modification, are permitted provided that the following conditions are6 * met:7 *8 * * Redistributions of source code must retain the above copyright9 * notice, this list of conditions and the following disclaimer.10 * * Redistributions in binary form must reproduce the above11 * copyright notice, this list of conditions and the following disclaimer12 * in the documentation and/or other materials provided with the13 * distribution.14 * * Neither the name of Google Inc. nor the names of its15 * contributors may be used to endorse or promote products derived from16 * this software without specific prior written permission.17 *18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.29 */3031// See http://groups.google.com/group/http-archive-specification/web/har-1-2-spec32// for HAR specification.3334// FIXME: Some fields are not yet supported due to back-end limitations.35// See https://bugs.webkit.org/show_bug.cgi?id=58127 for details.3637WebInspector.HAREntry = function(resource)38{39 this._resource = resource;40}4142WebInspector.HAREntry.prototype = {43 build: function()44 {45 return {46 pageref: this._resource.documentURL,47 startedDateTime: new Date(this._resource.startTime * 1000),48 time: WebInspector.HAREntry._toMilliseconds(this._resource.duration),49 request: this._buildRequest(),50 response: this._buildResponse(),51 cache: { }, // Not supported yet.52 timings: this._buildTimings()53 };54 },5556 _buildRequest: function()57 {58 var res = {59 method: this._resource.requestMethod,60 url: this._buildRequestURL(this._resource.url),61 httpVersion: this._resource.requestHttpVersion,62 headers: this._buildHeaders(this._resource.requestHeaders),63 queryString: this._buildParameters(this._resource.queryParameters || []),64 cookies: this._buildCookies(this._resource.requestCookies || []),65 headersSize: this._resource.requestHeadersSize,66 bodySize: this.requestBodySize67 };68 if (this._resource.requestFormData)69 res.postData = this._buildPostData();7071 return res;72 },7374 _buildResponse: function()75 {76 return {77 status: this._resource.statusCode,78 statusText: this._resource.statusText,79 httpVersion: this._resource.responseHttpVersion,80 headers: this._buildHeaders(this._resource.responseHeaders),81 cookies: this._buildCookies(this._resource.responseCookies || []),82 content: this._buildContent(),83 redirectURL: this._resource.responseHeaderValue("Location") || "",84 headersSize: this._resource.responseHeadersSize,85 bodySize: this.responseBodySize86 };87 },8889 _buildContent: function()90 {91 return {92 size: this._resource.resourceSize,93 compression: this.responseCompression,94 mimeType: this._resource.mimeType,95 // text: this._resource.content // TODO: pull out into a boolean flag, as content can be huge (and needs to be requested with an async call)96 };97 },9899 _buildTimings: function()100 {101 var waitForConnection = this._interval("connectStart", "connectEnd");102 var blocked;103 var connect;104 var dns = this._interval("dnsStart", "dnsEnd");105 var send = this._interval("sendStart", "sendEnd");106 var ssl = this._interval("sslStart", "sslEnd");107108 if (ssl !== -1 && send !== -1)109 send -= ssl;110111 if (this._resource.connectionReused) {112 connect = -1;113 blocked = waitForConnection;114 } else {115 blocked = 0;116 connect = waitForConnection;117 if (dns !== -1)118 connect -= dns;119 }120121 return {122 blocked: blocked,123 dns: dns,124 connect: connect,125 send: send,126 wait: this._interval("sendEnd", "receiveHeadersEnd"),127 receive: WebInspector.HAREntry._toMilliseconds(this._resource.receiveDuration),128 ssl: ssl129 };130 },131132 _buildHeaders: function(headers)133 {134 var result = [];135 for (var name in headers)136 result.push({ name: name, value: headers[name] });137 return result;138 },139140 _buildPostData: function()141 {142 var res = {143 mimeType: this._resource.requestHeaderValue("Content-Type"),144 text: this._resource.requestFormData145 };146 if (this._resource.formParameters)147 res.params = this._buildParameters(this._resource.formParameters);148 return res;149 },150151 _buildParameters: function(parameters)152 {153 return parameters.slice();154 },155156 _buildRequestURL: function(url)157 {158 return url.split("#", 2)[0];159 },160161 _buildCookies: function(cookies)162 {163 return cookies.map(this._buildCookie.bind(this));164 },165166 _buildCookie: function(cookie)167 {168 return {169 name: cookie.name,170 value: cookie.value,171 path: cookie.path,172 domain: cookie.domain,173 expires: cookie.expires(new Date(this._resource.startTime * 1000)),174 httpOnly: cookie.httpOnly,175 secure: cookie.secure176 };177 },178179 _interval: function(start, end)180 {181 var timing = this._resource.timing;182 if (!timing)183 return -1;184 var startTime = timing[start];185 return typeof startTime !== "number" || startTime === -1 ? -1 : Math.round(timing[end] - startTime);186 },187188 get requestBodySize()189 {190 return !this._resource.requestFormData ? 0 : this._resource.requestFormData.length;191 },192193 get responseBodySize()194 {195 return this._resource.transferSize - this._resource.responseHeadersSize196 },197198 get responseCompression()199 {200 return this._resource.resourceSize - (this._resource.transferSize - this._resource.responseHeadersSize);201 }202}203204WebInspector.HAREntry._toMilliseconds = function(time)205{206 return time === -1 ? -1 : Math.round(time * 1000);207}208209WebInspector.HARLog = function(resources)210{211 this._resources = resources;212}213214WebInspector.HARLog.prototype = {215 build: function()216 {217 var webKitVersion = /AppleWebKit\/([^ ]+)/.exec(window.navigator.userAgent);218219 return {220 version: "1.2",221 creator: {222 name: "WebInspector",223 version: webKitVersion ? webKitVersion[1] : "n/a"224 },225 pages: this._buildPages(),226 entries: this._resources.map(this._convertResource.bind(this))227 }228 },229230 _buildPages: function()231 {232 return [233 {234 startedDateTime: new Date(WebInspector.mainResource.startTime * 1000),235 id: WebInspector.mainResource.documentURL,236 title: "",237 pageTimings: this.buildMainResourceTimings()238 }239 ];240 },241242 buildMainResourceTimings: function()243 {244 return {245 onContentLoad: this._pageEventTime(WebInspector.mainResourceDOMContentTime),246 onLoad: this._pageEventTime(WebInspector.mainResourceLoadTime),247 }248 },249250 _convertResource: function(resource)251 {252 return (new WebInspector.HAREntry(resource)).build();253 },254255 _pageEventTime: function(time)256 {257 var startTime = WebInspector.mainResource.startTime;258 if (time === -1 || startTime === -1)259 return -1;260 return WebInspector.HAREntry._toMilliseconds(time - startTime);261 }
...
devtools.js
Source:devtools.js
1"use strict";2// devtools.js3console.log("REMORA: devtools/devtools.js loaded")4browser.devtools.panels.create(5 "Remora Investigator",6 "/icons/logo-green.png",7 "/devtools/main_panel/main.html",8).then((panel) => {9 console.log("PANEL Created!")10 console.log(panel)11 const theme = browser.devtools.panels.themeName12 if (theme === "dark") {13 console.log("Dark theme")14 } else {15 console.log("Other theme:", theme)16 }17 // code invoked on panel creation18})19const DATASTORE_SERVER = "http://localhost:65432"20const DATASTORE_ENDPOINT = "/api/save_event"21// Save Requests22const sendToDataStore = (message, callback) => {23 console.log("REMORA: devtools.js - sendToDataStore(): message=", message)24 fetch(`${DATASTORE_SERVER}${DATASTORE_ENDPOINT}`, {25 method: 'POST',26 mode: 'no-cors',27 headers: {28 'Accept': 'application/json',29 'Content-Type': 'application/json'30 },31 body: message32 }).then((response) => response.text())33 .then(callback)34}35browser.devtools.network.onRequestFinished.addListener((httpEvent) => {36 console.log("REMORA: devtools.js - onRequestFinished: httpEvent=", httpEvent)37 const url = httpEvent["request"]["url"]38 if ((url.includes("chrome://")) || (url.includes("moz-extension://") || url.includes(DATASTORE_SERVER))) {39 console.warn("REMORA: devtools.js - onRequestFinished (INTERNAL) httpEvent=", httpEvent)40 } else {41 httpEvent.getContent().then(([respBodyContent, respBodyEncoding]) => {42 // console.log("REMORA: devtools.js - onRequestFinished httpEvent.getContent: responseBodyContent ->", responseBodyContent)43 // console.log("REMORA: devtools.js - onRequestFinished httpEvent.getContent: requestBodyEncoding ->", requestBodyEncoding)44 // const message = JSON.parse(JSON.stringify(obj)) // * To avoid error: `"DOMException:" The object could not be cloned.`45 ///////////////////////////////////////////////////////////////////////////////////////46 // * Event metadata47 const eventSecurityState = httpEvent["_securityState"] || null48 const eventDestinationPort = parseInt(httpEvent["connection"]) || 049 const eventServerIPAddress = httpEvent["serverIPAddress"] || null50 const eventStartedDateTime = httpEvent["startedDateTime"] || null51 const eventRtt = parseInt(httpEvent["time"]) || 052 // * Request metadata 53 const requestMethod = httpEvent["request"]["method"]54 const requestUrl = httpEvent["request"]["url"]55 const requestHttpVersion = httpEvent["request"]["httpVersion"]56 const requestQueryString = JSON.stringify(httpEvent["request"]["queryString"]) // []57 const requestHeadersSize = httpEvent["request"]["headersSize"]58 const requestHeaders = httpEvent["request"]["headers"] // []59 const requestCookies = JSON.stringify(httpEvent["request"]["cookies"]) // []60 const requestBodySize = httpEvent["request"]["bodySize"]61 const isrequestHaspostData = httpEvent["request"].hasOwnProperty("postData")62 const requestPostDataMimeType = isrequestHaspostData ? httpEvent["request"]["postData"]["mimeType"] : null63 const requestPostDataParams = isrequestHaspostData ? JSON.stringify(httpEvent["request"]["postData"]["params"]) : null // []64 const requestPostDataText = isrequestHaspostData ? httpEvent["request"]["postData"]["text"] : null65 // * Response metadata66 const responseHttpVersion = httpEvent["response"]["httpVersion"]67 const responseStatusCode = httpEvent["response"]["status"]68 const responseHeadersSize = httpEvent["response"]["headersSize"]69 const responseHeaders = httpEvent["response"]["headers"] // []70 const responseRedirectURL = httpEvent["response"]["redirectURL"]71 const responseCookies = JSON.stringify(httpEvent["response"]["cookies"]) // []72 const responseContentMimeType = httpEvent["response"]["content"]["mimeType"]73 const responseContentSize = httpEvent["response"]["content"]["size"]74 const responseBodySize = httpEvent["response"]["content"]["size"] // TODO: You must check on this75 // const responseSize = httpEvent["response"]["bodySize"]76 const responseBodyEncoding = respBodyEncoding77 console.warn("typeof(responseBodyEncoding)=", typeof (responseBodyEncoding))78 // const isjavascriptCode = responseBodyEncoding === "application/javascript"79 // const responseBodyContent = isjavascriptCode ? btoa(`${respBodyContent}`) : respBodyContent80 // const responseBodyContent = btoa(`${respBodyContent}`)81 const responseBodyContent = `${respBodyContent}`82 console.warn("typeof(responseBodyContent)=", typeof (responseBodyContent))83 const message = {84 // Event85 security_state: eventSecurityState,86 destination_port: eventDestinationPort,87 server_ip_address: eventServerIPAddress,88 started_datetime: eventStartedDateTime,89 rtt: eventRtt,90 // Request91 request_method: requestMethod || null,92 request_url: requestUrl || null,93 request_http_version: requestHttpVersion || null,94 request_query_string: requestQueryString || null,95 request_headers_size: parseInt(requestHeadersSize) || 0,96 request_headers: requestHeaders || null,97 request_cookies: requestCookies || null,98 request_body_size: parseInt(requestBodySize) || 0,99 request_postdata_mimetype: requestPostDataMimeType || null,100 request_postdata_params: requestPostDataParams || null,101 request_postdata_text: requestPostDataText || null,102 // Response103 response_http_version: responseHttpVersion || null,104 response_status_code: parseInt(responseStatusCode) || 0,105 response_headers_size: parseInt(responseHeadersSize) || 0,106 response_headers: responseHeaders || null,107 response_redirect_url: responseRedirectURL || null,108 response_cookies: responseCookies || null,109 response_content_mimetype: responseContentMimeType || null,110 response_content_size: parseInt(responseContentSize) || 0,111 response_body_size: parseInt(responseBodySize) || 0,112 response_body_encoding: responseBodyEncoding || null,113 response_body_content: responseBodyContent,114 // Message metadata115 actor: "SaveHttpEvent",116 sender: "onRequestFinished",117 }118 // ///////////////////////////////////////////////////////////////////////////////////////119 // console.log("MESSAGE: message=", message)120 sendToDataStore(JSON.stringify(message), (response) => {121 // console.log("CALLBACK: response=", response)122 console.log("DATASTORE Reponse", response);123 })124 })125 }...
weibo.js
Source:weibo.js
1//@ts-check2const { chromium, errors } = require('playwright');3const { Client: PostgresClient } = require('pg');4const logger = require('pino')();5require('dotenv').config();6const REFRESH_THRESHOLD = parseInt(process.env.REFRESH_THRESHOLD, 10) || 166;7const RESTART_THRESHOLD = parseInt(process.env.RESTART_THRESHOLD, 10) || 30;8const PAGE_URL = process.env.PAGE_URL || 'https://www.weibo.com/hot/';9async function initDatabase() {10 const client = new PostgresClient({11 host: process.env.POSTGRES_HOST,12 user: process.env.POSTGRES_USER,13 password: process.env.POSTGRES_PASSWORD,14 database: process.env.POSTGRES_DATABASE,15 });16 await client.connect();17 logger.info('[Database] connected');18 return client;19}20async function launchBrowser() {21 const browser = await chromium.launch({22 headless: false,23 channel: 'msedge',24 args: ['--start-maximized']25 });26 const context = await browser.newContext({27 storageState: './weibo.json',28 viewport: null,29 });30 const page = await context.newPage();31 context.on('page', page => {32 page.close();33 })34 page.once('close', () => {35 browser.close();36 });37 logger.info('[Browser] launched');38 return page;39}40function observeNetworkActivities(page, dbClient) {41 page.on('requestfinished', async (req) => {42 try {43 const res = await req.response();44 const { requestBodySize, requestHeadersSize, responseBodySize, responseHeadersSize } = await req.sizes();45 const { startTime, domainLookupStart, domainLookupEnd, connectStart, secureConnectionStart, connectEnd, requestStart, responseStart, responseEnd } = req.timing();46 await dbClient.query(`47 INSERT INTO ipv6.request (url, resource_type, request_headers_size, request_body_size, 48 response_headers_size, response_body_size, ip, start_time, domain_lookup_start, 49 domain_lookup_end, connect_start, secure_connection_start, connect_end, 50 request_start, response_start, response_end, environment, domain)51 VALUES ($1, $2, $3, $4, $5, $6, $7, to_timestamp($8 / 1000.0), $9, $10, $11, $12, $13, $14, $15, $16, $17, $18)52 `, [53 req.url(),54 req.resourceType(),55 requestHeadersSize,56 requestBodySize,57 responseHeadersSize,58 responseBodySize,59 (await res.serverAddr())?.ipAddress,60 startTime,61 domainLookupStart,62 domainLookupEnd,63 connectStart,64 secureConnectionStart,65 connectEnd,66 requestStart,67 responseStart,68 responseEnd,69 process.env.IPV6_ENVIRONMENT,70 new URL(req.url()).hostname,71 ]);72 logger.debug(`[Request] ${req.url()}`);73 } catch (e) {74 logger.warn(`[Request] handling error`);75 }76 });77}78async function runTest(page) {79 const height = await page.evaluate('window.innerHeight');80 try {81 logger.info('[Refresh] start');82 await page.goto(PAGE_URL);83 logger.info('[Refresh] done');84 } catch (e) {85 // If the navigation is timeout, retry86 if (e instanceof errors.TimeoutError) {87 logger.warn('[Refresh] timeout');88 return;89 } else {90 throw e;91 }92 }93 94 for (let i = 0; i < REFRESH_THRESHOLD; i++) {95 const images = page.locator(`.woo-picture-slot:visible:below([role="navigation"], ${height})`);96 const firstImage = images.first();97 await firstImage.click({98 timeout: 1000,99 })100 .then(() => page.waitForSelector('svg[class^="CircleProgress"]', {101 state: 'attached',102 timeout: 1000,103 }))104 .then(() => page.waitForSelector('svg[class^="CircleProgress"]', {105 state: 'detached',106 timeout: 30000,107 }))108 .then(() => page.waitForTimeout(1000))109 .then(() => logger.info(`[ClickImage] ${i}/${REFRESH_THRESHOLD} clicked`))110 .catch(() => logger.info(`[ClickImage] ${i}/${REFRESH_THRESHOLD} timeout`));111 await page.keyboard.press('Escape');112 await page.waitForTimeout(500);113 await page.keyboard.press("PageDown");114 await page.waitForTimeout(1000);115 logger.info(`[Scrolling] ${i}/${REFRESH_THRESHOLD} done`);116 }117}118(async function loop() {119 const dbClient = await initDatabase();120 const page = await launchBrowser();121 observeNetworkActivities(page, dbClient);122 setTimeout(() => {123 process.exit();124 }, 1000 * 60 * RESTART_THRESHOLD);125 for (;;) {126 try {127 await runTest(page);128 } catch (e) {129 process.exit(1);130 }131 }132})().catch(() => {133 process.exit(1);...
kuaishou.js
Source:kuaishou.js
1//@ts-check2const { chromium, errors } = require('playwright');3const { Client: PostgresClient } = require('pg');4const logger = require('pino')();5require('dotenv').config();6const RESTART_THRESHOLD = parseInt(process.env.RESTART_THRESHOLD, 10) || 60;7const PAGE_URL = process.env.PAGE_URL || 'https://www.kuaishou.com/short-video/3xss4pn476pxzze';8async function initDatabase() {9 const client = new PostgresClient({10 host: process.env.POSTGRES_HOST,11 user: process.env.POSTGRES_USER,12 password: process.env.POSTGRES_PASSWORD,13 database: process.env.POSTGRES_DATABASE,14 });15 await client.connect();16 logger.info('[Database] connected');17 return client;18}19async function launchBrowser() {20 const browser = await chromium.launch({21 headless: false,22 channel: 'msedge',23 args: ['--start-maximized']24 });25 const context = await browser.newContext({26 viewport: null,27 });28 const page = await context.newPage();29 context.on('page', page => {30 page.close();31 })32 page.once('close', () => {33 browser.close();34 });35 logger.info('[Browser] launched');36 return page;37}38function observeNetworkActivities(page, dbClient) {39 page.on('requestfinished', async (req) => {40 const res = await req.response();41 const { requestBodySize, requestHeadersSize, responseBodySize, responseHeadersSize } = await req.sizes();42 const { startTime, domainLookupStart, domainLookupEnd, connectStart, secureConnectionStart, connectEnd, requestStart, responseStart, responseEnd } = req.timing();43 await dbClient.query(`44 INSERT INTO ipv6.request (url, resource_type, request_headers_size, request_body_size, 45 response_headers_size, response_body_size, ip, start_time, domain_lookup_start, 46 domain_lookup_end, connect_start, secure_connection_start, connect_end, 47 request_start, response_start, response_end, environment, domain)48 VALUES ($1, $2, $3, $4, $5, $6, $7, to_timestamp($8 / 1000.0), $9, $10, $11, $12, $13, $14, $15, $16, $17, $18)49 `, [50 req.url(),51 req.resourceType(),52 requestHeadersSize,53 requestBodySize,54 responseHeadersSize,55 responseBodySize,56 (await res.serverAddr())?.ipAddress,57 startTime,58 domainLookupStart,59 domainLookupEnd,60 connectStart,61 secureConnectionStart,62 connectEnd,63 requestStart,64 responseStart,65 responseEnd,66 process.env.IPV6_ENVIRONMENT,67 new URL(req.url()).hostname,68 ]);69 logger.debug(`[Request] ${req.url()}`);70 });71}72async function runTest(page) {73 try {74 logger.info('[Refresh] start');75 await page.goto(PAGE_URL);76 logger.info('[Refresh] done');77 await page.waitForTimeout(1000 * 60 * RESTART_THRESHOLD);78 } catch (e) {79 // If the navigation is timeout, retry80 if (e instanceof errors.TimeoutError) {81 logger.warn('[Refresh] timeout');82 return;83 } else {84 throw e;85 }86 }87}88(async function loop() {89 const dbClient = await initDatabase();90 const page = await launchBrowser();91 observeNetworkActivities(page, dbClient);92 let testStartTime = +new Date();93 for (;;) {94 try {95 if (+new Date() - testStartTime > 1000 * 60 * RESTART_THRESHOLD) {96 throw new Error('[Test] threshold reached');97 }98 await runTest(page);99 } catch (e) {100 await page.context().browser().close().catch(() => { });101 await dbClient.end().catch(() => { });102 logger.info('[Test] restarting browser');103 setTimeout(loop, 0);104 break;105 }106 }...
Using AI Code Generation
1const { chromium } = require('playwright');2(async () => {3 const browser = await chromium.launch();4 const context = await browser.newContext();5 const page = await context.newPage();6 console.log(await page.evaluate(() => window.performance.getEntriesByType('navigation')[0].requestHeadersSize));7 await browser.close();8})();9const { chromium } = require('playwright');10(async () => {11 const browser = await chromium.launch();12 const context = await browser.newContext();13 const page = await context.newPage();14 console.log(await page.evaluate(() => window.performance.getEntriesByType('navigation')[0].responseHeadersSize));15 await browser.close();16})();
Using AI Code Generation
1const { chromium } = require('playwright');2(async () => {3 const browser = await chromium.launch({headless: false});4 const context = await browser.newContext();5 const page = await context.newPage();6 console.log(request.headersSize());7 await browser.close();8})();9[MIT](LICENSE)
Using AI Code Generation
1const { chromium } = require('playwright');2(async () => {3 const browser = await chromium.launch();4 const context = await browser.newContext();5 const page = await context.newPage();6 const request = response.request();7 console.log(await request.headersSize());8 await browser.close();9})();
Using AI Code Generation
1const { chromium } = require('playwright');2(async () => {3 const browser = await chromium.launch({headless: false});4 const context = await browser.newContext();5 const page = await context.newPage();6 console.log(await page.evaluate(() => window.performance.getEntriesByType('resource')[0].requestHeadersSize));7 await browser.close();8})();9const { chromium } = require('playwright');10(async () => {11 const browser = await chromium.launch({headless: false});12 const context = await browser.newContext();13 const page = await context.newPage();14 await page.on('requestwillbesent', (request) => {15 console.log(request.headersSize);16 });17 await browser.close();18})();
Using AI Code Generation
1const playwright = require("playwright");2(async () => {3 for (const browserType of BROWSER) {4 const browser = await playwright[browserType].launch();5 const context = await browser.newContext();6 const page = await context.newPage();7 await page.route("**/*", (route, request) => {8 console.log(request.headers());9 route.continue();10 });11 await browser.close();12 }13})();14const playwright = require("playwright");15(async () => {16 for (const browserType of BROWSER) {17 const browser = await playwright[browserType].launch();18 const context = await browser.newContext();19 const page = await context.newPage();20 await page.route("**/*", (route, request) => {21 console.log(request.headers());22 route.continue();23 });24 await browser.close();25 }26})();27const playwright = require("playwright");28(async () => {29 for (const browserType of BROWSER) {30 const browser = await playwright[browserType].launch();31 const context = await browser.newContext();32 const page = await context.newPage();33 await page.route("**/*", (route, request) => {34 console.log(request.headers());35 route.continue();36 });37 await browser.close();38 }39})();40const playwright = require("playwright");41(async () => {42 for (const browserType of BROWSER) {43 const browser = await playwright[browserType].launch();44 const context = await browser.newContext();45 const page = await context.newPage();46 await page.route("**/*", (route, request) => {47 console.log(request.headers());48 route.continue();49 });50 await browser.close();51 }52})();53const playwright = require("playwright");54(async () => {55 for (const browserType of BROWSER) {
Using AI Code Generation
1const { chromium } = require('playwright');2(async () => {3 const browser = await chromium.launch();4 const context = await browser.newContext();5 const page = await context.newPage();6 await page.route('**', route => {7 const request = route.request();8 console.log(request.headers());9 console.log(request.requestHeadersSize());10 route.continue();11 });12 await browser.close();13})();14- [x] `browserType.launch([options])` - Launches a new browser instance. The browser will be closed when the parent node.js process is closed. If [persistent context](
Using AI Code Generation
1const { chromium } = require('playwright-chromium');2(async () => {3 const browser = await chromium.launch({ headless: false });4 const context = await browser.newContext();5 const page = await context.newPage();6 await page.route('**', route => {7 console.log(route.request().requestHeadersSize());8 route.continue();9 });10 await browser.close();11})();12const { chromium } = require('playwright-chromium');13(async () => {14 const browser = await chromium.launch({ headless: false });15 const context = await browser.newContext();16 const page = await context.newPage();17 await page.route('**', route => {18 console.log(route.request().responseHeadersSize());19 route.continue();20 });21 await browser.close();22})();23const { chromium } = require('playwright-chromium');24(async () => {25 const browser = await chromium.launch({ headless: false });26 const context = await browser.newContext();27 const page = await context.newPage();28 await page.route('**', route => {29 console.log(route.request().responseBodySize());30 route.continue();31 });32 await browser.close();33})();
Using AI Code Generation
1const { chromium } = require('playwright');2(async () => {3 const browser = await chromium.launch();4 const context = await browser.newContext();5 const page = await context.newPage();6 page.route('**', route => {7 const request = route.request();8 console.log(request.requestHeadersSize());9 route.continue();10 });11 await browser.close();12})();13const { chromium } = require('playwright');14(async () => {15 const browser = await chromium.launch();16 const context = await browser.newContext();17 const page = await context.newPage();18 page.route('**', route => {19 const request = route.request();20 console.log(request.requestHeadersSize());21 route.continue();22 });23 await browser.close();24})();25const { chromium } = require('playwright');26(async () => {27 const browser = await chromium.launch();28 const context = await browser.newContext();29 const page = await context.newPage();30 page.route('**', route => {31 const request = route.request();32 console.log(request.responseHeadersSize());
Using AI Code Generation
1const { chromium } = require('playwright-chromium');2const fs = require('fs');3const path = require('path');4const { promisify } = require('util');5const writeFile = promisify(fs.writeFile);6const { toMatchImageSnapshot } = require('jest-image-snapshot');7expect.extend({ toMatchImageSnapshot });8(async () => {9 const browser = await chromium.launch({ headless: false });10 const context = await browser.newContext();11 const page = await context.newPage();12 await page.goto('
LambdaTest’s Playwright tutorial will give you a broader idea about the Playwright automation framework, its unique features, and use cases with examples to exceed your understanding of Playwright testing. This tutorial will give A to Z guidance, from installing the Playwright framework to some best practices and advanced concepts.
Get 100 minutes of automation test minutes FREE!!