How to use ping method in localstack

Best Python code snippet using localstack_python

TelemetryStorage.jsm

Source:TelemetryStorage.jsm

1/* -*- js-indent-level: 2; indent-tabs-mode: nil -*- */2/* This Source Code Form is subject to the terms of the Mozilla Public3 * License, v. 2.0. If a copy of the MPL was not distributed with this4 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */5"use strict";6this.EXPORTED_SYMBOLS = ["TelemetryStorage"];7const Cc = Components.classes;8const Ci = Components.interfaces;9const Cr = Components.results;10const Cu = Components.utils;11Cu.import("resource://gre/modules/AppConstants.jsm", this);12Cu.import("resource://gre/modules/Log.jsm");13Cu.import("resource://gre/modules/Services.jsm", this);14Cu.import("resource://gre/modules/XPCOMUtils.jsm", this);15Cu.import("resource://gre/modules/osfile.jsm", this);16Cu.import("resource://gre/modules/Task.jsm", this);17Cu.import("resource://gre/modules/TelemetryUtils.jsm", this);18Cu.import("resource://gre/modules/Promise.jsm", this);19Cu.import("resource://gre/modules/Preferences.jsm", this);20const LOGGER_NAME = "Toolkit.Telemetry";21const LOGGER_PREFIX = "TelemetryStorage::";22const Telemetry = Services.telemetry;23const Utils = TelemetryUtils;24// Compute the path of the pings archive on the first use.25const DATAREPORTING_DIR = "datareporting";26const PINGS_ARCHIVE_DIR = "archived";27const ABORTED_SESSION_FILE_NAME = "aborted-session-ping";28const DELETION_PING_FILE_NAME = "pending-deletion-ping";29const SESSION_STATE_FILE_NAME = "session-state.json";30XPCOMUtils.defineLazyGetter(this, "gDataReportingDir", function() {31 return OS.Path.join(OS.Constants.Path.profileDir, DATAREPORTING_DIR);32});33XPCOMUtils.defineLazyGetter(this, "gPingsArchivePath", function() {34 return OS.Path.join(gDataReportingDir, PINGS_ARCHIVE_DIR);35});36XPCOMUtils.defineLazyGetter(this, "gAbortedSessionFilePath", function() {37 return OS.Path.join(gDataReportingDir, ABORTED_SESSION_FILE_NAME);38});39XPCOMUtils.defineLazyGetter(this, "gDeletionPingFilePath", function() {40 return OS.Path.join(gDataReportingDir, DELETION_PING_FILE_NAME);41});42XPCOMUtils.defineLazyModuleGetter(this, "CommonUtils",43 "resource://services-common/utils.js");44// Maxmimum time, in milliseconds, archive pings should be retained.45const MAX_ARCHIVED_PINGS_RETENTION_MS = 60 * 24 * 60 * 60 * 1000; // 60 days46// Maximum space the archive can take on disk (in Bytes).47const ARCHIVE_QUOTA_BYTES = 120 * 1024 * 1024; // 120 MB48// Maximum space the outgoing pings can take on disk, for Desktop (in Bytes).49const PENDING_PINGS_QUOTA_BYTES_DESKTOP = 15 * 1024 * 1024; // 15 MB50// Maximum space the outgoing pings can take on disk, for Mobile (in Bytes).51const PENDING_PINGS_QUOTA_BYTES_MOBILE = 1024 * 1024; // 1 MB52// The maximum size a pending/archived ping can take on disk.53const PING_FILE_MAXIMUM_SIZE_BYTES = 1024 * 1024; // 1 MB54// This special value is submitted when the archive is outside of the quota.55const ARCHIVE_SIZE_PROBE_SPECIAL_VALUE = 300;56// This special value is submitted when the pending pings is outside of the quota, as57// we don't know the size of the pings above the quota.58const PENDING_PINGS_SIZE_PROBE_SPECIAL_VALUE = 17;59const UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;60/**61 * This is thrown by |TelemetryStorage.loadPingFile| when reading the ping62 * from the disk fails.63 */64function PingReadError(message="Error reading the ping file", becauseNoSuchFile = false) {65 Error.call(this, message);66 let error = new Error();67 this.name = "PingReadError";68 this.message = message;69 this.stack = error.stack;70 this.becauseNoSuchFile = becauseNoSuchFile;71}72PingReadError.prototype = Object.create(Error.prototype);73PingReadError.prototype.constructor = PingReadError;74/**75 * This is thrown by |TelemetryStorage.loadPingFile| when parsing the ping JSON76 * content fails.77 */78function PingParseError(message="Error parsing ping content") {79 Error.call(this, message);80 let error = new Error();81 this.name = "PingParseError";82 this.message = message;83 this.stack = error.stack;84}85PingParseError.prototype = Object.create(Error.prototype);86PingParseError.prototype.constructor = PingParseError;87/**88 * This is a policy object used to override behavior for testing.89 */90var Policy = {91 now: () => new Date(),92 getArchiveQuota: () => ARCHIVE_QUOTA_BYTES,93 getPendingPingsQuota: () => (AppConstants.platform in ["android", "gonk"])94 ? PENDING_PINGS_QUOTA_BYTES_MOBILE95 : PENDING_PINGS_QUOTA_BYTES_DESKTOP,96};97/**98 * Wait for all promises in iterable to resolve or reject. This function99 * always resolves its promise with undefined, and never rejects.100 */101function waitForAll(it) {102 let dummy = () => {};103 let promises = Array.from(it, p => p.catch(dummy));104 return Promise.all(promises);105}106/**107 * Permanently intern the given string. This is mainly used for the ping.type108 * strings that can be excessively duplicated in the _archivedPings map. Do not109 * pass large or temporary strings to this function.110 */111function internString(str) {112 return Symbol.keyFor(Symbol.for(str));113}114this.TelemetryStorage = {115 get pingDirectoryPath() {116 return OS.Path.join(OS.Constants.Path.profileDir, "saved-telemetry-pings");117 },118 /**119 * The maximum size a ping can have, in bytes.120 */121 get MAXIMUM_PING_SIZE() {122 return PING_FILE_MAXIMUM_SIZE_BYTES;123 },124 /**125 * Shutdown & block on any outstanding async activity in this module.126 *127 * @return {Promise} Promise that is resolved when shutdown is complete.128 */129 shutdown: function() {130 return TelemetryStorageImpl.shutdown();131 },132 /**133 * Save an archived ping to disk.134 *135 * @param {object} ping The ping data to archive.136 * @return {promise} Promise that is resolved when the ping is successfully archived.137 */138 saveArchivedPing: function(ping) {139 return TelemetryStorageImpl.saveArchivedPing(ping);140 },141 /**142 * Load an archived ping from disk.143 *144 * @param {string} id The pings id.145 * @return {promise<object>} Promise that is resolved with the ping data.146 */147 loadArchivedPing: function(id) {148 return TelemetryStorageImpl.loadArchivedPing(id);149 },150 /**151 * Get a list of info on the archived pings.152 * This will scan the archive directory and grab basic data about the existing153 * pings out of their filename.154 *155 * @return {promise<sequence<object>>}156 */157 loadArchivedPingList: function() {158 return TelemetryStorageImpl.loadArchivedPingList();159 },160 /**161 * Clean the pings archive by removing old pings.162 * This will scan the archive directory.163 *164 * @return {Promise} Resolved when the cleanup task completes.165 */166 runCleanPingArchiveTask: function() {167 return TelemetryStorageImpl.runCleanPingArchiveTask();168 },169 /**170 * Run the task to enforce the pending pings quota.171 *172 * @return {Promise} Resolved when the cleanup task completes.173 */174 runEnforcePendingPingsQuotaTask: function() {175 return TelemetryStorageImpl.runEnforcePendingPingsQuotaTask();176 },177 /**178 * Run the task to remove all the pending pings (except the deletion ping).179 *180 * @return {Promise} Resolved when the pings are removed.181 */182 runRemovePendingPingsTask: function() {183 return TelemetryStorageImpl.runRemovePendingPingsTask();184 },185 /**186 * Reset the storage state in tests.187 */188 reset: function() {189 return TelemetryStorageImpl.reset();190 },191 /**192 * Test method that allows waiting on the archive clean task to finish.193 */194 testCleanupTaskPromise: function() {195 return (TelemetryStorageImpl._cleanArchiveTask || Promise.resolve());196 },197 /**198 * Test method that allows waiting on the pending pings quota task to finish.199 */200 testPendingQuotaTaskPromise: function() {201 return (TelemetryStorageImpl._enforcePendingPingsQuotaTask || Promise.resolve());202 },203 /**204 * Save a pending - outgoing - ping to disk and track it.205 *206 * @param {Object} ping The ping data.207 * @return {Promise} Resolved when the ping was saved.208 */209 savePendingPing: function(ping) {210 return TelemetryStorageImpl.savePendingPing(ping);211 },212 /**213 * Saves session data to disk.214 * @param {Object} sessionData The session data.215 * @return {Promise} Resolved when the data was saved.216 */217 saveSessionData: function(sessionData) {218 return TelemetryStorageImpl.saveSessionData(sessionData);219 },220 /**221 * Loads session data from a session data file.222 * @return {Promise<object>} Resolved with the session data in object form.223 */224 loadSessionData: function() {225 return TelemetryStorageImpl.loadSessionData();226 },227 /**228 * Load a pending ping from disk by id.229 *230 * @param {String} id The pings id.231 * @return {Promise} Resolved with the loaded ping data.232 */233 loadPendingPing: function(id) {234 return TelemetryStorageImpl.loadPendingPing(id);235 },236 /**237 * Remove a pending ping from disk by id.238 *239 * @param {String} id The pings id.240 * @return {Promise} Resolved when the ping was removed.241 */242 removePendingPing: function(id) {243 return TelemetryStorageImpl.removePendingPing(id);244 },245 /**246 * Returns a list of the currently pending pings in the format:247 * {248 * id: <string>, // The pings UUID.249 * lastModificationDate: <number>, // Timestamp of the pings last modification.250 * }251 * This populates the list by scanning the disk.252 *253 * @return {Promise<sequence>} Resolved with the ping list.254 */255 loadPendingPingList: function() {256 return TelemetryStorageImpl.loadPendingPingList();257 },258 /**259 * Returns a list of the currently pending pings in the format:260 * {261 * id: <string>, // The pings UUID.262 * lastModificationDate: <number>, // Timestamp of the pings last modification.263 * }264 * This does not scan pending pings on disk.265 *266 * @return {sequence} The current pending ping list.267 */268 getPendingPingList: function() {269 return TelemetryStorageImpl.getPendingPingList();270 },271 /**272 * Save an aborted-session ping to disk. This goes to a special location so273 * it is not picked up as a pending ping.274 *275 * @param {object} ping The ping data to save.276 * @return {promise} Promise that is resolved when the ping is successfully saved.277 */278 saveAbortedSessionPing: function(ping) {279 return TelemetryStorageImpl.saveAbortedSessionPing(ping);280 },281 /**282 * Load the aborted-session ping from disk if present.283 *284 * @return {promise<object>} Promise that is resolved with the ping data if found.285 * Otherwise returns null.286 */287 loadAbortedSessionPing: function() {288 return TelemetryStorageImpl.loadAbortedSessionPing();289 },290 /**291 * Save the deletion ping.292 * @param ping The deletion ping.293 * @return {Promise} A promise resolved when the ping is saved.294 */295 saveDeletionPing: function(ping) {296 return TelemetryStorageImpl.saveDeletionPing(ping);297 },298 /**299 * Remove the deletion ping.300 * @return {Promise} Resolved when the ping is deleted from the disk.301 */302 removeDeletionPing: function() {303 return TelemetryStorageImpl.removeDeletionPing();304 },305 /**306 * Check if the ping id identifies a deletion ping.307 */308 isDeletionPing: function(aPingId) {309 return TelemetryStorageImpl.isDeletionPing(aPingId);310 },311 /**312 * Remove the aborted-session ping if present.313 *314 * @return {promise} Promise that is resolved once the ping is removed.315 */316 removeAbortedSessionPing: function() {317 return TelemetryStorageImpl.removeAbortedSessionPing();318 },319 /**320 * Save a single ping to a file.321 *322 * @param {object} ping The content of the ping to save.323 * @param {string} file The destination file.324 * @param {bool} overwrite If |true|, the file will be overwritten if it exists,325 * if |false| the file will not be overwritten and no error will be reported if326 * the file exists.327 * @returns {promise}328 */329 savePingToFile: function(ping, file, overwrite) {330 return TelemetryStorageImpl.savePingToFile(ping, file, overwrite);331 },332 /**333 * Save a ping to its file.334 *335 * @param {object} ping The content of the ping to save.336 * @param {bool} overwrite If |true|, the file will be overwritten337 * if it exists.338 * @returns {promise}339 */340 savePing: function(ping, overwrite) {341 return TelemetryStorageImpl.savePing(ping, overwrite);342 },343 /**344 * Add a ping to the saved pings directory so that it gets saved345 * and sent along with other pings.346 *347 * @param {Object} pingData The ping object.348 * @return {Promise} A promise resolved when the ping is saved to the pings directory.349 */350 addPendingPing: function(pingData) {351 return TelemetryStorageImpl.addPendingPing(pingData);352 },353 /**354 * Remove the file for a ping355 *356 * @param {object} ping The ping.357 * @returns {promise}358 */359 cleanupPingFile: function(ping) {360 return TelemetryStorageImpl.cleanupPingFile(ping);361 },362 /**363 * The number of pending pings on disk.364 */365 get pendingPingCount() {366 return TelemetryStorageImpl.pendingPingCount;367 },368 /**369 * Loads a ping file.370 * @param {String} aFilePath The path of the ping file.371 * @return {Promise<Object>} A promise resolved with the ping content or rejected if the372 * ping contains invalid data.373 */374 loadPingFile: Task.async(function* (aFilePath) {375 return TelemetryStorageImpl.loadPingFile(aFilePath);376 }),377 /**378 * Remove FHR database files. This is temporary and will be dropped in379 * the future.380 * @return {Promise} Resolved when the database files are deleted.381 */382 removeFHRDatabase: function() {383 return TelemetryStorageImpl.removeFHRDatabase();384 },385 /**386 * Only used in tests, builds an archived ping path from the ping metadata.387 * @param {String} aPingId The ping id.388 * @param {Object} aDate The ping creation date.389 * @param {String} aType The ping type.390 * @return {String} The full path to the archived ping.391 */392 _testGetArchivedPingPath: function(aPingId, aDate, aType) {393 return getArchivedPingPath(aPingId, aDate, aType);394 },395 /**396 * Only used in tests, this helper extracts ping metadata from a given filename.397 *398 * @param fileName {String} The filename.399 * @return {Object} Null if the filename didn't match the expected form.400 * Otherwise an object with the extracted data in the form:401 * { timestamp: <number>,402 * id: <string>,403 * type: <string> }404 */405 _testGetArchivedPingDataFromFileName: function(aFileName) {406 return TelemetryStorageImpl._getArchivedPingDataFromFileName(aFileName);407 },408 /**409 * Only used in tests, this helper allows cleaning up the pending ping storage.410 */411 testClearPendingPings: function() {412 return TelemetryStorageImpl.runRemovePendingPingsTask();413 }414};415/**416 * This object allows the serialisation of asynchronous tasks. This is particularly417 * useful to serialise write access to the disk in order to prevent race conditions418 * to corrupt the data being written.419 * We are using this to synchronize saving to the file that TelemetrySession persists420 * its state in.421 */422function SaveSerializer() {423 this._queuedOperations = [];424 this._queuedInProgress = false;425 this._log = Log.repository.getLoggerWithMessagePrefix(LOGGER_NAME, LOGGER_PREFIX);426}427SaveSerializer.prototype = {428 /**429 * Enqueues an operation to a list to serialise their execution in order to prevent race430 * conditions. Useful to serialise access to disk.431 *432 * @param {Function} aFunction The task function to enqueue. It must return a promise.433 * @return {Promise} A promise resolved when the enqueued task completes.434 */435 enqueueTask: function (aFunction) {436 let promise = new Promise((resolve, reject) =>437 this._queuedOperations.push([aFunction, resolve, reject]));438 if (this._queuedOperations.length == 1) {439 this._popAndPerformQueuedOperation();440 }441 return promise;442 },443 /**444 * Make sure to flush all the pending operations.445 * @return {Promise} A promise resolved when all the pending operations have completed.446 */447 flushTasks: function () {448 let dummyTask = () => new Promise(resolve => resolve());449 return this.enqueueTask(dummyTask);450 },451 /**452 * Pop a task from the queue, executes it and continue to the next one.453 * This function recursively pops all the tasks.454 */455 _popAndPerformQueuedOperation: function () {456 if (!this._queuedOperations.length || this._queuedInProgress) {457 return;458 }459 this._log.trace("_popAndPerformQueuedOperation - Performing queued operation.");460 let [func, resolve, reject] = this._queuedOperations.shift();461 let promise;462 try {463 this._queuedInProgress = true;464 promise = func();465 } catch (ex) {466 this._log.warn("_popAndPerformQueuedOperation - Queued operation threw during execution. ",467 ex);468 this._queuedInProgress = false;469 reject(ex);470 this._popAndPerformQueuedOperation();471 return;472 }473 if (!promise || typeof(promise.then) != "function") {474 let msg = "Queued operation did not return a promise: " + func;475 this._log.warn("_popAndPerformQueuedOperation - " + msg);476 this._queuedInProgress = false;477 reject(new Error(msg));478 this._popAndPerformQueuedOperation();479 return;480 }481 promise.then(result => {482 this._queuedInProgress = false;483 resolve(result);484 this._popAndPerformQueuedOperation();485 },486 error => {487 this._log.warn("_popAndPerformQueuedOperation - Failure when performing queued operation.",488 error);489 this._queuedInProgress = false;490 reject(error);491 this._popAndPerformQueuedOperation();492 });493 },494};495var TelemetryStorageImpl = {496 _logger: null,497 // Used to serialize aborted session ping writes to disk.498 _abortedSessionSerializer: new SaveSerializer(),499 // Used to serialize deletion ping writes to disk.500 _deletionPingSerializer: new SaveSerializer(),501 // Used to serialize session state writes to disk.502 _stateSaveSerializer: new SaveSerializer(),503 // Tracks the archived pings in a Map of (id -> {timestampCreated, type}).504 // We use this to cache info on archived pings to avoid scanning the disk more than once.505 _archivedPings: new Map(),506 // A set of promises for pings currently being archived507 _activelyArchiving: new Set(),508 // Track the archive loading task to prevent multiple tasks from being executed.509 _scanArchiveTask: null,510 // Track the archive cleanup task.511 _cleanArchiveTask: null,512 // Whether we already scanned the archived pings on disk.513 _scannedArchiveDirectory: false,514 // Track the pending ping removal task.515 _removePendingPingsTask: null,516 // This tracks all the pending async ping save activity.517 _activePendingPingSaves: new Set(),518 // Tracks the pending pings in a Map of (id -> {timestampCreated, type}).519 // We use this to cache info on pending pings to avoid scanning the disk more than once.520 _pendingPings: new Map(),521 // Track the pending pings enforce quota task.522 _enforcePendingPingsQuotaTask: null,523 // Track the shutdown process to bail out of the clean up task quickly.524 _shutdown: false,525 get _log() {526 if (!this._logger) {527 this._logger = Log.repository.getLoggerWithMessagePrefix(LOGGER_NAME, LOGGER_PREFIX);528 }529 return this._logger;530 },531 /**532 * Shutdown & block on any outstanding async activity in this module.533 *534 * @return {Promise} Promise that is resolved when shutdown is complete.535 */536 shutdown: Task.async(function*() {537 this._shutdown = true;538 // If the following tasks are still running, block on them. They will bail out as soon539 // as possible.540 yield this._abortedSessionSerializer.flushTasks().catch(ex => {541 this._log.error("shutdown - failed to flush aborted-session writes", ex);542 });543 yield this._deletionPingSerializer.flushTasks().catch(ex => {544 this._log.error("shutdown - failed to flush deletion ping writes", ex);545 });546 if (this._cleanArchiveTask) {547 yield this._cleanArchiveTask.catch(ex => {548 this._log.error("shutdown - the archive cleaning task failed", ex);549 });550 }551 if (this._enforcePendingPingsQuotaTask) {552 yield this._enforcePendingPingsQuotaTask.catch(ex => {553 this._log.error("shutdown - the pending pings quota task failed", ex);554 });555 }556 if (this._removePendingPingsTask) {557 yield this._removePendingPingsTask.catch(ex => {558 this._log.error("shutdown - the pending pings removal task failed", ex);559 });560 }561 // Wait on pending pings still being saved. While OS.File should have shutdown562 // blockers in place, we a) have seen weird errors being reported that might563 // indicate a bad shutdown path and b) might have completion handlers hanging564 // off the save operations that don't expect to be late in shutdown.565 yield this.promisePendingPingSaves();566 }),567 /**568 * Save an archived ping to disk.569 *570 * @param {object} ping The ping data to archive.571 * @return {promise} Promise that is resolved when the ping is successfully archived.572 */573 saveArchivedPing: function(ping) {574 let promise = this._saveArchivedPingTask(ping);575 this._activelyArchiving.add(promise);576 promise.then((r) => { this._activelyArchiving.delete(promise); },577 (e) => { this._activelyArchiving.delete(promise); });578 return promise;579 },580 _saveArchivedPingTask: Task.async(function*(ping) {581 const creationDate = new Date(ping.creationDate);582 if (this._archivedPings.has(ping.id)) {583 const data = this._archivedPings.get(ping.id);584 if (data.timestampCreated > creationDate.getTime()) {585 this._log.error("saveArchivedPing - trying to overwrite newer ping with the same id");586 return Promise.reject(new Error("trying to overwrite newer ping with the same id"));587 }588 this._log.warn("saveArchivedPing - overwriting older ping with the same id");589 }590 // Get the archived ping path and append the lz4 suffix to it (so we have 'jsonlz4').591 const filePath = getArchivedPingPath(ping.id, creationDate, ping.type) + "lz4";592 yield OS.File.makeDir(OS.Path.dirname(filePath), { ignoreExisting: true,593 from: OS.Constants.Path.profileDir });594 yield this.savePingToFile(ping, filePath, /* overwrite*/ true, /* compressed*/ true);595 this._archivedPings.set(ping.id, {596 timestampCreated: creationDate.getTime(),597 type: internString(ping.type),598 });599 Telemetry.getHistogramById("TELEMETRY_ARCHIVE_SESSION_PING_COUNT").add();600 return undefined;601 }),602 /**603 * Load an archived ping from disk.604 *605 * @param {string} id The pings id.606 * @return {promise<object>} Promise that is resolved with the ping data.607 */608 loadArchivedPing: Task.async(function*(id) {609 this._log.trace("loadArchivedPing - id: " + id);610 const data = this._archivedPings.get(id);611 if (!data) {612 this._log.trace("loadArchivedPing - no ping with id: " + id);613 return Promise.reject(new Error("TelemetryStorage.loadArchivedPing - no ping with id " + id));614 }615 const path = getArchivedPingPath(id, new Date(data.timestampCreated), data.type);616 const pathCompressed = path + "lz4";617 // Purge pings which are too big.618 let checkSize = function*(path) {619 const fileSize = (yield OS.File.stat(path)).size;620 if (fileSize > PING_FILE_MAXIMUM_SIZE_BYTES) {621 Telemetry.getHistogramById("TELEMETRY_DISCARDED_ARCHIVED_PINGS_SIZE_MB")622 .add(Math.floor(fileSize / 1024 / 1024));623 Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_ARCHIVED").add();624 yield OS.File.remove(path, {ignoreAbsent: true});625 throw new Error("loadArchivedPing - exceeded the maximum ping size: " + fileSize);626 }627 };628 try {629 // Try to load a compressed version of the archived ping first.630 this._log.trace("loadArchivedPing - loading ping from: " + pathCompressed);631 yield* checkSize(pathCompressed);632 return yield this.loadPingFile(pathCompressed, /* compressed*/ true);633 } catch (ex) {634 if (!ex.becauseNoSuchFile) {635 throw ex;636 }637 // If that fails, look for the uncompressed version.638 this._log.trace("loadArchivedPing - compressed ping not found, loading: " + path);639 yield* checkSize(path);640 return yield this.loadPingFile(path, /* compressed*/ false);641 }642 }),643 /**644 * Saves session data to disk.645 */646 saveSessionData: function(sessionData) {647 return this._stateSaveSerializer.enqueueTask(() => this._saveSessionData(sessionData));648 },649 _saveSessionData: Task.async(function* (sessionData) {650 let dataDir = OS.Path.join(OS.Constants.Path.profileDir, DATAREPORTING_DIR);651 yield OS.File.makeDir(dataDir);652 let filePath = OS.Path.join(gDataReportingDir, SESSION_STATE_FILE_NAME);653 try {654 yield CommonUtils.writeJSON(sessionData, filePath);655 } catch (e) {656 this._log.error("_saveSessionData - Failed to write session data to " + filePath, e);657 Telemetry.getHistogramById("TELEMETRY_SESSIONDATA_FAILED_SAVE").add(1);658 }659 }),660 /**661 * Loads session data from the session data file.662 * @return {Promise<Object>} A promise resolved with an object on success,663 * with null otherwise.664 */665 loadSessionData: function() {666 return this._stateSaveSerializer.enqueueTask(() => this._loadSessionData());667 },668 _loadSessionData: Task.async(function* () {669 const dataFile = OS.Path.join(OS.Constants.Path.profileDir, DATAREPORTING_DIR,670 SESSION_STATE_FILE_NAME);671 let content;672 try {673 content = yield OS.File.read(dataFile, { encoding: "utf-8" });674 } catch (ex) {675 this._log.info("_loadSessionData - can not load session data file", ex);676 Telemetry.getHistogramById("TELEMETRY_SESSIONDATA_FAILED_LOAD").add(1);677 return null;678 }679 let data;680 try {681 data = JSON.parse(content);682 } catch (ex) {683 this._log.error("_loadSessionData - failed to parse session data", ex);684 Telemetry.getHistogramById("TELEMETRY_SESSIONDATA_FAILED_PARSE").add(1);685 return null;686 }687 return data;688 }),689 /**690 * Remove an archived ping from disk.691 *692 * @param {string} id The pings id.693 * @param {number} timestampCreated The pings creation timestamp.694 * @param {string} type The pings type.695 * @return {promise<object>} Promise that is resolved when the pings is removed.696 */697 _removeArchivedPing: Task.async(function*(id, timestampCreated, type) {698 this._log.trace("_removeArchivedPing - id: " + id + ", timestampCreated: " + timestampCreated + ", type: " + type);699 const path = getArchivedPingPath(id, new Date(timestampCreated), type);700 const pathCompressed = path + "lz4";701 this._log.trace("_removeArchivedPing - removing ping from: " + path);702 yield OS.File.remove(path, {ignoreAbsent: true});703 yield OS.File.remove(pathCompressed, {ignoreAbsent: true});704 // Remove the ping from the cache.705 this._archivedPings.delete(id);706 }),707 /**708 * Clean the pings archive by removing old pings.709 *710 * @return {Promise} Resolved when the cleanup task completes.711 */712 runCleanPingArchiveTask: function() {713 // If there's an archive cleaning task already running, return it.714 if (this._cleanArchiveTask) {715 return this._cleanArchiveTask;716 }717 // Make sure to clear |_cleanArchiveTask| once done.718 let clear = () => this._cleanArchiveTask = null;719 // Since there's no archive cleaning task running, start it.720 this._cleanArchiveTask = this._cleanArchive().then(clear, clear);721 return this._cleanArchiveTask;722 },723 /**724 * Removes pings which are too old from the pings archive.725 * @return {Promise} Resolved when the ping age check is complete.726 */727 _purgeOldPings: Task.async(function*() {728 this._log.trace("_purgeOldPings");729 const nowDate = Policy.now();730 const startTimeStamp = nowDate.getTime();731 let dirIterator = new OS.File.DirectoryIterator(gPingsArchivePath);732 let subdirs = (yield dirIterator.nextBatch()).filter(e => e.isDir);733 dirIterator.close();734 // Keep track of the newest removed month to update the cache, if needed.735 let newestRemovedMonthTimestamp = null;736 let evictedDirsCount = 0;737 let maxDirAgeInMonths = 0;738 // Walk through the monthly subdirs of the form <YYYY-MM>/739 for (let dir of subdirs) {740 if (this._shutdown) {741 this._log.trace("_purgeOldPings - Terminating the clean up task due to shutdown");742 return;743 }744 if (!isValidArchiveDir(dir.name)) {745 this._log.warn("_purgeOldPings - skipping invalidly named subdirectory " + dir.path);746 continue;747 }748 const archiveDate = getDateFromArchiveDir(dir.name);749 if (!archiveDate) {750 this._log.warn("_purgeOldPings - skipping invalid subdirectory date " + dir.path);751 continue;752 }753 // If this archive directory is older than 180 days, remove it.754 if ((startTimeStamp - archiveDate.getTime()) > MAX_ARCHIVED_PINGS_RETENTION_MS) {755 try {756 yield OS.File.removeDir(dir.path);757 evictedDirsCount++;758 // Update the newest removed month.759 newestRemovedMonthTimestamp = Math.max(archiveDate, newestRemovedMonthTimestamp);760 } catch (ex) {761 this._log.error("_purgeOldPings - Unable to remove " + dir.path, ex);762 }763 } else {764 // We're not removing this directory, so record the age for the oldest directory.765 const dirAgeInMonths = Utils.getElapsedTimeInMonths(archiveDate, nowDate);766 maxDirAgeInMonths = Math.max(dirAgeInMonths, maxDirAgeInMonths);767 }768 }769 // Trigger scanning of the archived pings.770 yield this.loadArchivedPingList();771 // Refresh the cache: we could still skip this, but it's cheap enough to keep it772 // to avoid introducing task dependencies.773 if (newestRemovedMonthTimestamp) {774 // Scan the archive cache for pings older than the newest directory pruned above.775 for (let [id, info] of this._archivedPings) {776 const timestampCreated = new Date(info.timestampCreated);777 if (timestampCreated.getTime() > newestRemovedMonthTimestamp) {778 continue;779 }780 // Remove outdated pings from the cache.781 this._archivedPings.delete(id);782 }783 }784 const endTimeStamp = Policy.now().getTime();785 // Save the time it takes to evict old directories and the eviction count.786 Telemetry.getHistogramById("TELEMETRY_ARCHIVE_EVICTED_OLD_DIRS")787 .add(evictedDirsCount);788 Telemetry.getHistogramById("TELEMETRY_ARCHIVE_EVICTING_DIRS_MS")789 .add(Math.ceil(endTimeStamp - startTimeStamp));790 Telemetry.getHistogramById("TELEMETRY_ARCHIVE_OLDEST_DIRECTORY_AGE")791 .add(maxDirAgeInMonths);792 }),793 /**794 * Enforce a disk quota for the pings archive.795 * @return {Promise} Resolved when the quota check is complete.796 */797 _enforceArchiveQuota: Task.async(function*() {798 this._log.trace("_enforceArchiveQuota");799 let startTimeStamp = Policy.now().getTime();800 // Build an ordered list, from newer to older, of archived pings.801 let pingList = Array.from(this._archivedPings, p => ({802 id: p[0],803 timestampCreated: p[1].timestampCreated,804 type: p[1].type,805 }));806 pingList.sort((a, b) => b.timestampCreated - a.timestampCreated);807 // If our archive is too big, we should reduce it to reach 90% of the quota.808 const SAFE_QUOTA = Policy.getArchiveQuota() * 0.9;809 // The index of the last ping to keep. Pings older than this one will be deleted if810 // the archive exceeds the quota.811 let lastPingIndexToKeep = null;812 let archiveSizeInBytes = 0;813 // Find the disk size of the archive.814 for (let i = 0; i < pingList.length; i++) {815 if (this._shutdown) {816 this._log.trace("_enforceArchiveQuota - Terminating the clean up task due to shutdown");817 return;818 }819 let ping = pingList[i];820 // Get the size for this ping.821 const fileSize =822 yield getArchivedPingSize(ping.id, new Date(ping.timestampCreated), ping.type);823 if (!fileSize) {824 this._log.warn("_enforceArchiveQuota - Unable to find the size of ping " + ping.id);825 continue;826 }827 // Enforce a maximum file size limit on archived pings.828 if (fileSize > PING_FILE_MAXIMUM_SIZE_BYTES) {829 this._log.error("_enforceArchiveQuota - removing file exceeding size limit, size: " + fileSize);830 // We just remove the ping from the disk, we don't bother removing it from pingList831 // since it won't contribute to the quota.832 yield this._removeArchivedPing(ping.id, ping.timestampCreated, ping.type)833 .catch(e => this._log.error("_enforceArchiveQuota - failed to remove archived ping" + ping.id));834 Telemetry.getHistogramById("TELEMETRY_DISCARDED_ARCHIVED_PINGS_SIZE_MB")835 .add(Math.floor(fileSize / 1024 / 1024));836 Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_ARCHIVED").add();837 continue;838 }839 archiveSizeInBytes += fileSize;840 if (archiveSizeInBytes < SAFE_QUOTA) {841 // We save the index of the last ping which is ok to keep in order to speed up ping842 // pruning.843 lastPingIndexToKeep = i;844 } else if (archiveSizeInBytes > Policy.getArchiveQuota()) {845 // Ouch, our ping archive is too big. Bail out and start pruning!846 break;847 }848 }849 // Save the time it takes to check if the archive is over-quota.850 Telemetry.getHistogramById("TELEMETRY_ARCHIVE_CHECKING_OVER_QUOTA_MS")851 .add(Math.round(Policy.now().getTime() - startTimeStamp));852 let submitProbes = (sizeInMB, evictedPings, elapsedMs) => {853 Telemetry.getHistogramById("TELEMETRY_ARCHIVE_SIZE_MB").add(sizeInMB);854 Telemetry.getHistogramById("TELEMETRY_ARCHIVE_EVICTED_OVER_QUOTA").add(evictedPings);855 Telemetry.getHistogramById("TELEMETRY_ARCHIVE_EVICTING_OVER_QUOTA_MS").add(elapsedMs);856 };857 // Check if we're using too much space. If not, submit the archive size and bail out.858 if (archiveSizeInBytes < Policy.getArchiveQuota()) {859 submitProbes(Math.round(archiveSizeInBytes / 1024 / 1024), 0, 0);860 return;861 }862 this._log.info("_enforceArchiveQuota - archive size: " + archiveSizeInBytes + "bytes"863 + ", safety quota: " + SAFE_QUOTA + "bytes");864 startTimeStamp = Policy.now().getTime();865 let pingsToPurge = pingList.slice(lastPingIndexToKeep + 1);866 // Remove all the pings older than the last one which we are safe to keep.867 for (let ping of pingsToPurge) {868 if (this._shutdown) {869 this._log.trace("_enforceArchiveQuota - Terminating the clean up task due to shutdown");870 return;871 }872 // This list is guaranteed to be in order, so remove the pings at its873 // beginning (oldest).874 yield this._removeArchivedPing(ping.id, ping.timestampCreated, ping.type);875 }876 const endTimeStamp = Policy.now().getTime();877 submitProbes(ARCHIVE_SIZE_PROBE_SPECIAL_VALUE, pingsToPurge.length,878 Math.ceil(endTimeStamp - startTimeStamp));879 }),880 _cleanArchive: Task.async(function*() {881 this._log.trace("cleanArchiveTask");882 if (!(yield OS.File.exists(gPingsArchivePath))) {883 return;884 }885 // Remove pings older than 180 days.886 try {887 yield this._purgeOldPings();888 } catch (ex) {889 this._log.error("_cleanArchive - There was an error removing old directories", ex);890 }891 // Make sure we respect the archive disk quota.892 yield this._enforceArchiveQuota();893 }),894 /**895 * Run the task to enforce the pending pings quota.896 *897 * @return {Promise} Resolved when the cleanup task completes.898 */899 runEnforcePendingPingsQuotaTask: Task.async(function*() {900 // If there's a cleaning task already running, return it.901 if (this._enforcePendingPingsQuotaTask) {902 return this._enforcePendingPingsQuotaTask;903 }904 // Since there's no quota enforcing task running, start it.905 try {906 this._enforcePendingPingsQuotaTask = this._enforcePendingPingsQuota();907 yield this._enforcePendingPingsQuotaTask;908 } finally {909 this._enforcePendingPingsQuotaTask = null;910 }911 return undefined;912 }),913 /**914 * Enforce a disk quota for the pending pings.915 * @return {Promise} Resolved when the quota check is complete.916 */917 _enforcePendingPingsQuota: Task.async(function*() {918 this._log.trace("_enforcePendingPingsQuota");919 let startTimeStamp = Policy.now().getTime();920 // Build an ordered list, from newer to older, of pending pings.921 let pingList = Array.from(this._pendingPings, p => ({922 id: p[0],923 lastModificationDate: p[1].lastModificationDate,924 }));925 pingList.sort((a, b) => b.lastModificationDate - a.lastModificationDate);926 // If our pending pings directory is too big, we should reduce it to reach 90% of the quota.927 const SAFE_QUOTA = Policy.getPendingPingsQuota() * 0.9;928 // The index of the last ping to keep. Pings older than this one will be deleted if929 // the pending pings directory size exceeds the quota.930 let lastPingIndexToKeep = null;931 let pendingPingsSizeInBytes = 0;932 // Find the disk size of the pending pings directory.933 for (let i = 0; i < pingList.length; i++) {934 if (this._shutdown) {935 this._log.trace("_enforcePendingPingsQuota - Terminating the clean up task due to shutdown");936 return;937 }938 let ping = pingList[i];939 // Get the size for this ping.940 const fileSize = yield getPendingPingSize(ping.id);941 if (!fileSize) {942 this._log.warn("_enforcePendingPingsQuota - Unable to find the size of ping " + ping.id);943 continue;944 }945 pendingPingsSizeInBytes += fileSize;946 if (pendingPingsSizeInBytes < SAFE_QUOTA) {947 // We save the index of the last ping which is ok to keep in order to speed up ping948 // pruning.949 lastPingIndexToKeep = i;950 } else if (pendingPingsSizeInBytes > Policy.getPendingPingsQuota()) {951 // Ouch, our pending pings directory size is too big. Bail out and start pruning!952 break;953 }954 }955 // Save the time it takes to check if the pending pings are over-quota.956 Telemetry.getHistogramById("TELEMETRY_PENDING_CHECKING_OVER_QUOTA_MS")957 .add(Math.round(Policy.now().getTime() - startTimeStamp));958 let recordHistograms = (sizeInMB, evictedPings, elapsedMs) => {959 Telemetry.getHistogramById("TELEMETRY_PENDING_PINGS_SIZE_MB").add(sizeInMB);960 Telemetry.getHistogramById("TELEMETRY_PENDING_PINGS_EVICTED_OVER_QUOTA").add(evictedPings);961 Telemetry.getHistogramById("TELEMETRY_PENDING_EVICTING_OVER_QUOTA_MS").add(elapsedMs);962 };963 // Check if we're using too much space. If not, bail out.964 if (pendingPingsSizeInBytes < Policy.getPendingPingsQuota()) {965 recordHistograms(Math.round(pendingPingsSizeInBytes / 1024 / 1024), 0, 0);966 return;967 }968 this._log.info("_enforcePendingPingsQuota - size: " + pendingPingsSizeInBytes + "bytes"969 + ", safety quota: " + SAFE_QUOTA + "bytes");970 startTimeStamp = Policy.now().getTime();971 let pingsToPurge = pingList.slice(lastPingIndexToKeep + 1);972 // Remove all the pings older than the last one which we are safe to keep.973 for (let ping of pingsToPurge) {974 if (this._shutdown) {975 this._log.trace("_enforcePendingPingsQuota - Terminating the clean up task due to shutdown");976 return;977 }978 // This list is guaranteed to be in order, so remove the pings at its979 // beginning (oldest).980 yield this.removePendingPing(ping.id);981 }982 const endTimeStamp = Policy.now().getTime();983 // We don't know the size of the pending pings directory if we are above the quota,984 // since we stop scanning once we reach the quota. We use a special value to show985 // this condition.986 recordHistograms(PENDING_PINGS_SIZE_PROBE_SPECIAL_VALUE, pingsToPurge.length,987 Math.ceil(endTimeStamp - startTimeStamp));988 }),989 /**990 * Reset the storage state in tests.991 */992 reset: function() {993 this._shutdown = false;994 this._scannedArchiveDirectory = false;995 this._archivedPings = new Map();996 this._scannedPendingDirectory = false;997 this._pendingPings = new Map();998 },999 /**1000 * Get a list of info on the archived pings.1001 * This will scan the archive directory and grab basic data about the existing1002 * pings out of their filename.1003 *1004 * @return {promise<sequence<object>>}1005 */1006 loadArchivedPingList: Task.async(function*() {1007 // If there's an archive loading task already running, return it.1008 if (this._scanArchiveTask) {1009 return this._scanArchiveTask;1010 }1011 yield waitForAll(this._activelyArchiving);1012 if (this._scannedArchiveDirectory) {1013 this._log.trace("loadArchivedPingList - Archive already scanned, hitting cache.");1014 return this._archivedPings;1015 }1016 // Since there's no archive loading task running, start it.1017 let result;1018 try {1019 this._scanArchiveTask = this._scanArchive();1020 result = yield this._scanArchiveTask;1021 } finally {1022 this._scanArchiveTask = null;1023 }1024 return result;1025 }),1026 _scanArchive: Task.async(function*() {1027 this._log.trace("_scanArchive");1028 let submitProbes = (pingCount, dirCount) => {1029 Telemetry.getHistogramById("TELEMETRY_ARCHIVE_SCAN_PING_COUNT")1030 .add(pingCount);1031 Telemetry.getHistogramById("TELEMETRY_ARCHIVE_DIRECTORIES_COUNT")1032 .add(dirCount);1033 };1034 if (!(yield OS.File.exists(gPingsArchivePath))) {1035 submitProbes(0, 0);1036 return new Map();1037 }1038 let dirIterator = new OS.File.DirectoryIterator(gPingsArchivePath);1039 let subdirs =1040 (yield dirIterator.nextBatch()).filter(e => e.isDir).filter(e => isValidArchiveDir(e.name));1041 dirIterator.close();1042 // Walk through the monthly subdirs of the form <YYYY-MM>/1043 for (let dir of subdirs) {1044 this._log.trace("_scanArchive - checking in subdir: " + dir.path);1045 let pingIterator = new OS.File.DirectoryIterator(dir.path);1046 let pings = (yield pingIterator.nextBatch()).filter(e => !e.isDir);1047 pingIterator.close();1048 // Now process any ping files of the form "<timestamp>.<uuid>.<type>.[json|jsonlz4]".1049 for (let p of pings) {1050 // data may be null if the filename doesn't match the above format.1051 let data = this._getArchivedPingDataFromFileName(p.name);1052 if (!data) {1053 continue;1054 }1055 // In case of conflicts, overwrite only with newer pings.1056 if (this._archivedPings.has(data.id)) {1057 const overwrite = data.timestamp > this._archivedPings.get(data.id).timestampCreated;1058 this._log.warn("_scanArchive - have seen this id before: " + data.id +1059 ", overwrite: " + overwrite);1060 if (!overwrite) {1061 continue;1062 }1063 yield this._removeArchivedPing(data.id, data.timestampCreated, data.type)1064 .catch((e) => this._log.warn("_scanArchive - failed to remove ping", e));1065 }1066 this._archivedPings.set(data.id, {1067 timestampCreated: data.timestamp,1068 type: internString(data.type),1069 });1070 }1071 }1072 // Mark the archive as scanned, so we no longer hit the disk.1073 this._scannedArchiveDirectory = true;1074 // Update the ping and directories count histograms.1075 submitProbes(this._archivedPings.size, subdirs.length);1076 return this._archivedPings;1077 }),1078 /**1079 * Save a single ping to a file.1080 *1081 * @param {object} ping The content of the ping to save.1082 * @param {string} file The destination file.1083 * @param {bool} overwrite If |true|, the file will be overwritten if it exists,1084 * if |false| the file will not be overwritten and no error will be reported if1085 * the file exists.1086 * @param {bool} [compress=false] If |true|, the file will use lz4 compression. Otherwise no1087 * compression will be used.1088 * @returns {promise}1089 */1090 savePingToFile: Task.async(function*(ping, filePath, overwrite, compress = false) {1091 try {1092 this._log.trace("savePingToFile - path: " + filePath);1093 let pingString = JSON.stringify(ping);1094 let options = { tmpPath: filePath + ".tmp", noOverwrite: !overwrite };1095 if (compress) {1096 options.compression = "lz4";1097 }1098 yield OS.File.writeAtomic(filePath, pingString, options);1099 } catch (e) {1100 if (!e.becauseExists) {1101 throw e;1102 }1103 }1104 }),1105 /**1106 * Save a ping to its file.1107 *1108 * @param {object} ping The content of the ping to save.1109 * @param {bool} overwrite If |true|, the file will be overwritten1110 * if it exists.1111 * @returns {promise}1112 */1113 savePing: Task.async(function*(ping, overwrite) {1114 yield getPingDirectory();1115 let file = pingFilePath(ping);1116 yield this.savePingToFile(ping, file, overwrite);1117 return file;1118 }),1119 /**1120 * Add a ping to the saved pings directory so that it gets saved1121 * and sent along with other pings.1122 * Note: that the original ping file will not be modified.1123 *1124 * @param {Object} ping The ping object.1125 * @return {Promise} A promise resolved when the ping is saved to the pings directory.1126 */1127 addPendingPing: function(ping) {1128 return this.savePendingPing(ping);1129 },1130 /**1131 * Remove the file for a ping1132 *1133 * @param {object} ping The ping.1134 * @returns {promise}1135 */1136 cleanupPingFile: function(ping) {1137 return OS.File.remove(pingFilePath(ping));1138 },1139 savePendingPing: function(ping) {1140 let p = this.savePing(ping, true).then((path) => {1141 this._pendingPings.set(ping.id, {1142 path: path,1143 lastModificationDate: Policy.now().getTime(),1144 });1145 this._log.trace("savePendingPing - saved ping with id " + ping.id);1146 });1147 this._trackPendingPingSaveTask(p);1148 return p;1149 },1150 loadPendingPing: Task.async(function*(id) {1151 this._log.trace("loadPendingPing - id: " + id);1152 let info = this._pendingPings.get(id);1153 if (!info) {1154 this._log.trace("loadPendingPing - unknown id " + id);1155 throw new Error("TelemetryStorage.loadPendingPing - no ping with id " + id);1156 }1157 // Try to get the dimension of the ping. If that fails, update the histograms.1158 let fileSize = 0;1159 try {1160 fileSize = (yield OS.File.stat(info.path)).size;1161 } catch (e) {1162 if (!(e instanceof OS.File.Error) || !e.becauseNoSuchFile) {1163 throw e;1164 }1165 // Fall through and let |loadPingFile| report the error.1166 }1167 // Purge pings which are too big.1168 if (fileSize > PING_FILE_MAXIMUM_SIZE_BYTES) {1169 yield this.removePendingPing(id);1170 Telemetry.getHistogramById("TELEMETRY_DISCARDED_PENDING_PINGS_SIZE_MB")1171 .add(Math.floor(fileSize / 1024 / 1024));1172 Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_PENDING").add();1173 throw new Error("loadPendingPing - exceeded the maximum ping size: " + fileSize);1174 }1175 // Try to load the ping file. Update the related histograms on failure.1176 let ping;1177 try {1178 ping = yield this.loadPingFile(info.path, false);1179 } catch (e) {1180 // If we failed to load the ping, check what happened and update the histogram.1181 if (e instanceof PingReadError) {1182 Telemetry.getHistogramById("TELEMETRY_PENDING_LOAD_FAILURE_READ").add();1183 } else if (e instanceof PingParseError) {1184 Telemetry.getHistogramById("TELEMETRY_PENDING_LOAD_FAILURE_PARSE").add();1185 }1186 // Remove the ping from the cache, so we don't try to load it again.1187 this._pendingPings.delete(id);1188 // Then propagate the rejection.1189 throw e;1190 }1191 return ping;1192 }),1193 removePendingPing: function(id) {1194 let info = this._pendingPings.get(id);1195 if (!info) {1196 this._log.trace("removePendingPing - unknown id " + id);1197 return Promise.resolve();1198 }1199 this._log.trace("removePendingPing - deleting ping with id: " + id +1200 ", path: " + info.path);1201 this._pendingPings.delete(id);1202 return OS.File.remove(info.path).catch((ex) =>1203 this._log.error("removePendingPing - failed to remove ping", ex));1204 },1205 /**1206 * Track any pending ping save tasks through the promise passed here.1207 * This is needed to block on any outstanding ping save activity.1208 *1209 * @param {Object<Promise>} The save promise to track.1210 */1211 _trackPendingPingSaveTask: function (promise) {1212 let clear = () => this._activePendingPingSaves.delete(promise);1213 promise.then(clear, clear);1214 this._activePendingPingSaves.add(promise);1215 },1216 /**1217 * Return a promise that allows to wait on pending pings being saved.1218 * @return {Object<Promise>} A promise resolved when all the pending pings save promises1219 * are resolved.1220 */1221 promisePendingPingSaves: function () {1222 // Make sure to wait for all the promises, even if they reject. We don't need to log1223 // the failures here, as they are already logged elsewhere.1224 return waitForAll(this._activePendingPingSaves);1225 },1226 /**1227 * Run the task to remove all the pending pings (except the deletion ping).1228 *1229 * @return {Promise} Resolved when the pings are removed.1230 */1231 runRemovePendingPingsTask: Task.async(function*() {1232 // If we already have a pending pings removal task active, return that.1233 if (this._removePendingPingsTask) {1234 return this._removePendingPingsTask;1235 }1236 // Start the task to remove all pending pings. Also make sure to clear the task once done.1237 try {1238 this._removePendingPingsTask = this.removePendingPings();1239 yield this._removePendingPingsTask;1240 } finally {1241 this._removePendingPingsTask = null;1242 }1243 return undefined;1244 }),1245 removePendingPings: Task.async(function*() {1246 this._log.trace("removePendingPings - removing all pending pings");1247 // Wait on pending pings still being saved, so so we don't miss removing them.1248 yield this.promisePendingPingSaves();1249 // Individually remove existing pings, so we don't interfere with operations expecting1250 // the pending pings directory to exist.1251 const directory = TelemetryStorage.pingDirectoryPath;1252 let iter = new OS.File.DirectoryIterator(directory);1253 try {1254 if (!(yield iter.exists())) {1255 this._log.trace("removePendingPings - the pending pings directory doesn't exist");1256 return;1257 }1258 let files = (yield iter.nextBatch()).filter(e => !e.isDir);1259 for (let file of files) {1260 try {1261 yield OS.File.remove(file.path);1262 } catch (ex) {1263 this._log.error("removePendingPings - failed to remove file " + file.path, ex);1264 continue;1265 }1266 }1267 } finally {1268 yield iter.close();1269 }1270 }),1271 loadPendingPingList: function() {1272 // If we already have a pending scanning task active, return that.1273 if (this._scanPendingPingsTask) {1274 return this._scanPendingPingsTask;1275 }1276 if (this._scannedPendingDirectory) {1277 this._log.trace("loadPendingPingList - Pending already scanned, hitting cache.");1278 return Promise.resolve(this._buildPingList());1279 }1280 // Since there's no pending pings scan task running, start it.1281 // Also make sure to clear the task once done.1282 this._scanPendingPingsTask = this._scanPendingPings().then(pings => {1283 this._scanPendingPingsTask = null;1284 return pings;1285 }, ex => {1286 this._scanPendingPingsTask = null;1287 throw ex;1288 });1289 return this._scanPendingPingsTask;1290 },1291 getPendingPingList: function() {1292 return this._buildPingList();1293 },1294 _scanPendingPings: Task.async(function*() {1295 this._log.trace("_scanPendingPings");1296 let directory = TelemetryStorage.pingDirectoryPath;1297 let iter = new OS.File.DirectoryIterator(directory);1298 let exists = yield iter.exists();1299 try {1300 if (!exists) {1301 return [];1302 }1303 let files = (yield iter.nextBatch()).filter(e => !e.isDir);1304 for (let file of files) {1305 if (this._shutdown) {1306 return [];1307 }1308 let info;1309 try {1310 info = yield OS.File.stat(file.path);1311 } catch (ex) {1312 this._log.error("_scanPendingPings - failed to stat file " + file.path, ex);1313 continue;1314 }1315 // Enforce a maximum file size limit on pending pings.1316 if (info.size > PING_FILE_MAXIMUM_SIZE_BYTES) {1317 this._log.error("_scanPendingPings - removing file exceeding size limit " + file.path);1318 try {1319 yield OS.File.remove(file.path);1320 } catch (ex) {1321 this._log.error("_scanPendingPings - failed to remove file " + file.path, ex);1322 } finally {1323 Telemetry.getHistogramById("TELEMETRY_DISCARDED_PENDING_PINGS_SIZE_MB")1324 .add(Math.floor(info.size / 1024 / 1024));1325 Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_PENDING").add();1326 continue;1327 }1328 }1329 let id = OS.Path.basename(file.path);1330 if (!UUID_REGEX.test(id)) {1331 this._log.trace("_scanPendingPings - filename is not a UUID: " + id);1332 id = Utils.generateUUID();1333 }1334 this._pendingPings.set(id, {1335 path: file.path,1336 lastModificationDate: info.lastModificationDate.getTime(),1337 });1338 }1339 } finally {1340 yield iter.close();1341 }1342 // Explicitly load the deletion ping from its known path, if it's there.1343 if (yield OS.File.exists(gDeletionPingFilePath)) {1344 this._log.trace("_scanPendingPings - Adding pending deletion ping.");1345 // We can't get the ping id or the last modification date without hitting the disk.1346 // Since deletion has a special handling, we don't really need those.1347 this._pendingPings.set(Utils.generateUUID(), {1348 path: gDeletionPingFilePath,1349 lastModificationDate: Date.now(),1350 });1351 }1352 this._scannedPendingDirectory = true;1353 return this._buildPingList();1354 }),1355 _buildPingList: function() {1356 const list = Array.from(this._pendingPings, p => ({1357 id: p[0],1358 lastModificationDate: p[1].lastModificationDate,1359 }));1360 list.sort((a, b) => b.lastModificationDate - a.lastModificationDate);1361 return list;1362 },1363 get pendingPingCount() {1364 return this._pendingPings.size;1365 },1366 /**1367 * Loads a ping file.1368 * @param {String} aFilePath The path of the ping file.1369 * @param {Boolean} [aCompressed=false] If |true|, expects the file to be compressed using lz4.1370 * @return {Promise<Object>} A promise resolved with the ping content or rejected if the1371 * ping contains invalid data.1372 * @throws {PingReadError} There was an error while reading the ping file from the disk.1373 * @throws {PingParseError} There was an error while parsing the JSON content of the ping file.1374 */1375 loadPingFile: Task.async(function* (aFilePath, aCompressed = false) {1376 let options = {};1377 if (aCompressed) {1378 options.compression = "lz4";1379 }1380 let array;1381 try {1382 array = yield OS.File.read(aFilePath, options);1383 } catch (e) {1384 this._log.trace("loadPingfile - unreadable ping " + aFilePath, e);1385 throw new PingReadError(e.message, e.becauseNoSuchFile);1386 }1387 let decoder = new TextDecoder();1388 let string = decoder.decode(array);1389 let ping;1390 try {1391 ping = JSON.parse(string);1392 } catch (e) {1393 this._log.trace("loadPingfile - unparseable ping " + aFilePath, e);1394 yield OS.File.remove(aFilePath).catch((ex) => {1395 this._log.error("loadPingFile - failed removing unparseable ping file", ex);1396 });1397 throw new PingParseError(e.message);1398 }1399 return ping;1400 }),1401 /**1402 * Archived pings are saved with file names of the form:1403 * "<timestamp>.<uuid>.<type>.[json|jsonlz4]"1404 * This helper extracts that data from a given filename.1405 *1406 * @param fileName {String} The filename.1407 * @return {Object} Null if the filename didn't match the expected form.1408 * Otherwise an object with the extracted data in the form:1409 * { timestamp: <number>,1410 * id: <string>,1411 * type: <string> }1412 */1413 _getArchivedPingDataFromFileName: function(fileName) {1414 // Extract the parts.1415 let parts = fileName.split(".");1416 if (parts.length != 4) {1417 this._log.trace("_getArchivedPingDataFromFileName - should have 4 parts");1418 return null;1419 }1420 let [timestamp, uuid, type, extension] = parts;1421 if (extension != "json" && extension != "jsonlz4") {1422 this._log.trace("_getArchivedPingDataFromFileName - should have 'json' or 'jsonlz4' extension");1423 return null;1424 }1425 // Check for a valid timestamp.1426 timestamp = parseInt(timestamp);1427 if (Number.isNaN(timestamp)) {1428 this._log.trace("_getArchivedPingDataFromFileName - should have a valid timestamp");1429 return null;1430 }1431 // Check for a valid UUID.1432 if (!UUID_REGEX.test(uuid)) {1433 this._log.trace("_getArchivedPingDataFromFileName - should have a valid id");1434 return null;1435 }1436 // Check for a valid type string.1437 const typeRegex = /^[a-z0-9][a-z0-9-]+[a-z0-9]$/i;1438 if (!typeRegex.test(type)) {1439 this._log.trace("_getArchivedPingDataFromFileName - should have a valid type");1440 return null;1441 }1442 return {1443 timestamp: timestamp,1444 id: uuid,1445 type: type,1446 };1447 },1448 saveAbortedSessionPing: Task.async(function*(ping) {1449 this._log.trace("saveAbortedSessionPing - ping path: " + gAbortedSessionFilePath);1450 yield OS.File.makeDir(gDataReportingDir, { ignoreExisting: true });1451 return this._abortedSessionSerializer.enqueueTask(() =>1452 this.savePingToFile(ping, gAbortedSessionFilePath, true));1453 }),1454 loadAbortedSessionPing: Task.async(function*() {1455 let ping = null;1456 try {1457 ping = yield this.loadPingFile(gAbortedSessionFilePath);1458 } catch (ex) {1459 if (ex.becauseNoSuchFile) {1460 this._log.trace("loadAbortedSessionPing - no such file");1461 } else {1462 this._log.error("loadAbortedSessionPing - error loading ping", ex)1463 }1464 }1465 return ping;1466 }),1467 removeAbortedSessionPing: function() {1468 return this._abortedSessionSerializer.enqueueTask(Task.async(function*() {1469 try {1470 yield OS.File.remove(gAbortedSessionFilePath, { ignoreAbsent: false });1471 this._log.trace("removeAbortedSessionPing - success");1472 } catch (ex) {1473 if (ex.becauseNoSuchFile) {1474 this._log.trace("removeAbortedSessionPing - no such file");1475 } else {1476 this._log.error("removeAbortedSessionPing - error removing ping", ex)1477 }1478 }1479 }.bind(this)));1480 },1481 /**1482 * Save the deletion ping.1483 * @param ping The deletion ping.1484 * @return {Promise} Resolved when the ping is saved.1485 */1486 saveDeletionPing: Task.async(function*(ping) {1487 this._log.trace("saveDeletionPing - ping path: " + gDeletionPingFilePath);1488 yield OS.File.makeDir(gDataReportingDir, { ignoreExisting: true });1489 let p = this._deletionPingSerializer.enqueueTask(() =>1490 this.savePingToFile(ping, gDeletionPingFilePath, true));1491 this._trackPendingPingSaveTask(p);1492 return p;1493 }),1494 /**1495 * Remove the deletion ping.1496 * @return {Promise} Resolved when the ping is deleted from the disk.1497 */1498 removeDeletionPing: Task.async(function*() {1499 return this._deletionPingSerializer.enqueueTask(Task.async(function*() {1500 try {1501 yield OS.File.remove(gDeletionPingFilePath, { ignoreAbsent: false });1502 this._log.trace("removeDeletionPing - success");1503 } catch (ex) {1504 if (ex.becauseNoSuchFile) {1505 this._log.trace("removeDeletionPing - no such file");1506 } else {1507 this._log.error("removeDeletionPing - error removing ping", ex)1508 }1509 }1510 }.bind(this)));1511 }),1512 isDeletionPing: function(aPingId) {1513 this._log.trace("isDeletionPing - id: " + aPingId);1514 let pingInfo = this._pendingPings.get(aPingId);1515 if (!pingInfo) {1516 return false;1517 }1518 if (pingInfo.path != gDeletionPingFilePath) {1519 return false;1520 }1521 return true;1522 },1523 /**1524 * Remove FHR database files. This is temporary and will be dropped in1525 * the future.1526 * @return {Promise} Resolved when the database files are deleted.1527 */1528 removeFHRDatabase: Task.async(function*() {1529 this._log.trace("removeFHRDatabase");1530 // Let's try to remove the FHR DB with the default filename first.1531 const FHR_DB_DEFAULT_FILENAME = "healthreport.sqlite";1532 // Even if it's uncommon, there may be 2 additional files: - a "write ahead log"1533 // (-wal) file and a "shared memory file" (-shm). We need to remove them as well.1534 let FILES_TO_REMOVE = [1535 OS.Path.join(OS.Constants.Path.profileDir, FHR_DB_DEFAULT_FILENAME),1536 OS.Path.join(OS.Constants.Path.profileDir, FHR_DB_DEFAULT_FILENAME + "-wal"),1537 OS.Path.join(OS.Constants.Path.profileDir, FHR_DB_DEFAULT_FILENAME + "-shm"),1538 ];1539 // FHR could have used either the default DB file name or a custom one1540 // through this preference.1541 const FHR_DB_CUSTOM_FILENAME =1542 Preferences.get("datareporting.healthreport.dbName", undefined);1543 if (FHR_DB_CUSTOM_FILENAME) {1544 FILES_TO_REMOVE.push(1545 OS.Path.join(OS.Constants.Path.profileDir, FHR_DB_CUSTOM_FILENAME),1546 OS.Path.join(OS.Constants.Path.profileDir, FHR_DB_CUSTOM_FILENAME + "-wal"),1547 OS.Path.join(OS.Constants.Path.profileDir, FHR_DB_CUSTOM_FILENAME + "-shm"));1548 }1549 for (let f of FILES_TO_REMOVE) {1550 yield OS.File.remove(f, {ignoreAbsent: true})1551 .catch(e => this._log.error("removeFHRDatabase - failed to remove " + f, e));1552 }1553 }),1554};1555// Utility functions1556function pingFilePath(ping) {1557 // Support legacy ping formats, who don't have an "id" field, but a "slug" field.1558 let pingIdentifier = (ping.slug) ? ping.slug : ping.id;1559 return OS.Path.join(TelemetryStorage.pingDirectoryPath, pingIdentifier);1560}1561function getPingDirectory() {1562 return Task.spawn(function*() {1563 let directory = TelemetryStorage.pingDirectoryPath;1564 if (!(yield OS.File.exists(directory))) {1565 yield OS.File.makeDir(directory, { unixMode: OS.Constants.S_IRWXU });1566 }1567 return directory;1568 });1569}1570/**1571 * Build the path to the archived ping.1572 * @param {String} aPingId The ping id.1573 * @param {Object} aDate The ping creation date.1574 * @param {String} aType The ping type.1575 * @return {String} The full path to the archived ping.1576 */1577function getArchivedPingPath(aPingId, aDate, aType) {1578 // Helper to pad the month to 2 digits, if needed (e.g. "1" -> "01").1579 let addLeftPadding = value => (value < 10) ? ("0" + value) : value;1580 // Get the ping creation date and generate the archive directory to hold it. Note1581 // that getMonth returns a 0-based month, so we need to add an offset.1582 let archivedPingDir = OS.Path.join(gPingsArchivePath,1583 aDate.getFullYear() + '-' + addLeftPadding(aDate.getMonth() + 1));1584 // Generate the archived ping file path as YYYY-MM/<TIMESTAMP>.UUID.type.json1585 let fileName = [aDate.getTime(), aPingId, aType, "json"].join(".");1586 return OS.Path.join(archivedPingDir, fileName);1587}1588/**1589 * Get the size of the ping file on the disk.1590 * @return {Integer} The file size, in bytes, of the ping file or 0 on errors.1591 */1592var getArchivedPingSize = Task.async(function*(aPingId, aDate, aType) {1593 const path = getArchivedPingPath(aPingId, aDate, aType);1594 let filePaths = [ path + "lz4", path ];1595 for (let path of filePaths) {1596 try {1597 return (yield OS.File.stat(path)).size;1598 } catch (e) {}1599 }1600 // That's odd, this ping doesn't seem to exist.1601 return 0;1602});1603/**1604 * Get the size of the pending ping file on the disk.1605 * @return {Integer} The file size, in bytes, of the ping file or 0 on errors.1606 */1607var getPendingPingSize = Task.async(function*(aPingId) {1608 const path = OS.Path.join(TelemetryStorage.pingDirectoryPath, aPingId)1609 try {1610 return (yield OS.File.stat(path)).size;1611 } catch (e) {}1612 // That's odd, this ping doesn't seem to exist.1613 return 0;1614});1615/**1616 * Check if a directory name is in the "YYYY-MM" format.1617 * @param {String} aDirName The name of the pings archive directory.1618 * @return {Boolean} True if the directory name is in the right format, false otherwise.1619 */1620function isValidArchiveDir(aDirName) {1621 const dirRegEx = /^[0-9]{4}-[0-9]{2}$/;1622 return dirRegEx.test(aDirName);1623}1624/**1625 * Gets a date object from an archive directory name.1626 * @param {String} aDirName The name of the pings archive directory. Must be in the YYYY-MM1627 * format.1628 * @return {Object} A Date object or null if the dir name is not valid.1629 */1630function getDateFromArchiveDir(aDirName) {1631 let [year, month] = aDirName.split("-");1632 year = parseInt(year);1633 month = parseInt(month);1634 // Make sure to have sane numbers.1635 if (!Number.isFinite(month) || !Number.isFinite(year) || month < 1 || month > 12) {1636 return null;1637 }1638 return new Date(year, month - 1, 1, 0, 0, 0);...

test_TelemetrySendOldPings.js

Source:test_TelemetrySendOldPings.js

1/* Any copyright is dedicated to the Public Domain.2 http://creativecommons.org/publicdomain/zero/1.0/3/**4 * This test case populates the profile with some fake stored5 * pings, and checks that pending pings are immediatlely sent6 * after delayed init.7 */8"use strict"9Cu.import("resource://gre/modules/osfile.jsm", this);10Cu.import("resource://gre/modules/Services.jsm", this);11Cu.import("resource://gre/modules/Promise.jsm", this);12Cu.import("resource://gre/modules/TelemetryStorage.jsm", this);13Cu.import("resource://gre/modules/TelemetryController.jsm", this);14Cu.import("resource://gre/modules/TelemetrySend.jsm", this);15Cu.import("resource://gre/modules/Task.jsm", this);16Cu.import("resource://gre/modules/XPCOMUtils.jsm");17var {OS: {File, Path, Constants}} = Cu.import("resource://gre/modules/osfile.jsm", {});18// We increment TelemetryStorage's MAX_PING_FILE_AGE and19// OVERDUE_PING_FILE_AGE by 1 minute so that our test pings exceed20// those points in time, even taking into account file system imprecision.21const ONE_MINUTE_MS = 60 * 1000;22const OVERDUE_PING_FILE_AGE = TelemetrySend.OVERDUE_PING_FILE_AGE + ONE_MINUTE_MS;23const PING_SAVE_FOLDER = "saved-telemetry-pings";24const PING_TIMEOUT_LENGTH = 5000;25const OVERDUE_PINGS = 6;26const OLD_FORMAT_PINGS = 4;27const RECENT_PINGS = 4;28const TOTAL_EXPECTED_PINGS = OVERDUE_PINGS + RECENT_PINGS + OLD_FORMAT_PINGS;29const PREF_FHR_UPLOAD = "datareporting.healthreport.uploadEnabled";30var gCreatedPings = 0;31var gSeenPings = 0;32/**33 * Creates some Telemetry pings for the and saves them to disk. Each ping gets a34 * unique ID based on an incrementor.35 *36 * @param {Array} aPingInfos An array of ping type objects. Each entry must be an37 * object containing a "num" field for the number of pings to create and38 * an "age" field. The latter representing the age in milliseconds to offset39 * from now. A value of 10 would make the ping 10ms older than now, for40 * example.41 * @returns Promise42 * @resolve an Array with the created pings ids.43 */44var createSavedPings = Task.async(function* (aPingInfos) {45 let pingIds = [];46 let now = Date.now();47 for (let type in aPingInfos) {48 let num = aPingInfos[type].num;49 let age = now - (aPingInfos[type].age || 0);50 for (let i = 0; i < num; ++i) {51 let pingId = yield TelemetryController.addPendingPing("test-ping", {}, { overwrite: true });52 if (aPingInfos[type].age) {53 // savePing writes to the file synchronously, so we're good to54 // modify the lastModifedTime now.55 let filePath = getSavePathForPingId(pingId);56 yield File.setDates(filePath, null, age);57 }58 gCreatedPings++;59 pingIds.push(pingId);60 }61 }62 return pingIds;63});64/**65 * Deletes locally saved pings if they exist.66 *67 * @param aPingIds an Array of ping ids to delete.68 * @returns Promise69 */70var clearPings = Task.async(function* (aPingIds) {71 for (let pingId of aPingIds) {72 yield TelemetryStorage.removePendingPing(pingId);73 }74});75/**76 * Fakes the pending pings storage quota.77 * @param {Integer} aPendingQuota The new quota, in bytes.78 */79function fakePendingPingsQuota(aPendingQuota) {80 let storage = Cu.import("resource://gre/modules/TelemetryStorage.jsm");81 storage.Policy.getPendingPingsQuota = () => aPendingQuota;82}83/**84 * Returns a handle for the file that a ping should be85 * stored in locally.86 *87 * @returns path88 */89function getSavePathForPingId(aPingId) {90 return Path.join(Constants.Path.profileDir, PING_SAVE_FOLDER, aPingId);91}92/**93 * Check if the number of Telemetry pings received by the HttpServer is not equal94 * to aExpectedNum.95 *96 * @param aExpectedNum the number of pings we expect to receive.97 */98function assertReceivedPings(aExpectedNum) {99 do_check_eq(gSeenPings, aExpectedNum);100}101/**102 * Throws if any pings with the id in aPingIds is saved locally.103 *104 * @param aPingIds an Array of pings ids to check.105 * @returns Promise106 */107var assertNotSaved = Task.async(function* (aPingIds) {108 let saved = 0;109 for (let id of aPingIds) {110 let filePath = getSavePathForPingId(id);111 if (yield File.exists(filePath)) {112 saved++;113 }114 }115 if (saved > 0) {116 do_throw("Found " + saved + " unexpected saved pings.");117 }118});119/**120 * Our handler function for the HttpServer that simply121 * increments the gSeenPings global when it successfully122 * receives and decodes a Telemetry payload.123 *124 * @param aRequest the HTTP request sent from HttpServer.125 */126function pingHandler(aRequest) {127 gSeenPings++;128}129add_task(function* test_setup() {130 PingServer.start();131 PingServer.registerPingHandler(pingHandler);132 do_get_profile();133 loadAddonManager("xpcshell@tests.mozilla.org", "XPCShell", "1", "1.9.2");134 // Make sure we don't generate unexpected pings due to pref changes.135 yield setEmptyPrefWatchlist();136 Services.prefs.setBoolPref(PREF_TELEMETRY_ENABLED, true);137 Services.prefs.setCharPref(TelemetryController.Constants.PREF_SERVER,138 "http://localhost:" + PingServer.port);139});140/**141 * Setup the tests by making sure the ping storage directory is available, otherwise142 * |TelemetryController.testSaveDirectoryToFile| could fail.143 */144add_task(function* setupEnvironment() {145 // The following tests assume this pref to be true by default.146 Services.prefs.setBoolPref(PREF_FHR_UPLOAD, true);147 yield TelemetryController.testSetup();148 let directory = TelemetryStorage.pingDirectoryPath;149 yield File.makeDir(directory, { ignoreExisting: true, unixMode: OS.Constants.S_IRWXU });150 yield TelemetryStorage.testClearPendingPings();151});152/**153 * Test that really recent pings are sent on Telemetry initialization.154 */155add_task(function* test_recent_pings_sent() {156 let pingTypes = [{ num: RECENT_PINGS }];157 yield createSavedPings(pingTypes);158 yield TelemetryController.testReset();159 yield TelemetrySend.testWaitOnOutgoingPings();160 assertReceivedPings(RECENT_PINGS);161 yield TelemetryStorage.testClearPendingPings();162});163/**164 * Create an overdue ping in the old format and try to send it.165 */166add_task(function* test_overdue_old_format() {167 // A test ping in the old, standard format.168 const PING_OLD_FORMAT = {169 slug: "1234567abcd",170 reason: "test-ping",171 payload: {172 info: {173 reason: "test-ping",174 OS: "XPCShell",175 appID: "SomeId",176 appVersion: "1.0",177 appName: "XPCShell",178 appBuildID: "123456789",179 appUpdateChannel: "Test",180 platformBuildID: "987654321",181 },182 },183 };184 // A ping with no info section, but with a slug.185 const PING_NO_INFO = {186 slug: "1234-no-info-ping",187 reason: "test-ping",188 payload: {}189 };190 // A ping with no payload.191 const PING_NO_PAYLOAD = {192 slug: "5678-no-payload",193 reason: "test-ping",194 };195 // A ping with no info and no slug.196 const PING_NO_SLUG = {197 reason: "test-ping",198 payload: {}199 };200 const PING_FILES_PATHS = [201 getSavePathForPingId(PING_OLD_FORMAT.slug),202 getSavePathForPingId(PING_NO_INFO.slug),203 getSavePathForPingId(PING_NO_PAYLOAD.slug),204 getSavePathForPingId("no-slug-file"),205 ];206 // Write the ping to file and make it overdue.207 yield TelemetryStorage.savePing(PING_OLD_FORMAT, true);208 yield TelemetryStorage.savePing(PING_NO_INFO, true);209 yield TelemetryStorage.savePing(PING_NO_PAYLOAD, true);210 yield TelemetryStorage.savePingToFile(PING_NO_SLUG, PING_FILES_PATHS[3], true);211 for (let f in PING_FILES_PATHS) {212 yield File.setDates(PING_FILES_PATHS[f], null, Date.now() - OVERDUE_PING_FILE_AGE);213 }214 gSeenPings = 0;215 yield TelemetryController.testReset();216 yield TelemetrySend.testWaitOnOutgoingPings();217 assertReceivedPings(OLD_FORMAT_PINGS);218 // |TelemetryStorage.cleanup| doesn't know how to remove a ping with no slug or id,219 // so remove it manually so that the next test doesn't fail.220 yield OS.File.remove(PING_FILES_PATHS[3]);221 yield TelemetryStorage.testClearPendingPings();222});223add_task(function* test_corrupted_pending_pings() {224 const TEST_TYPE = "test_corrupted";225 Telemetry.getHistogramById("TELEMETRY_PENDING_LOAD_FAILURE_READ").clear();226 Telemetry.getHistogramById("TELEMETRY_PENDING_LOAD_FAILURE_PARSE").clear();227 // Save a pending ping and get its id.228 let pendingPingId = yield TelemetryController.addPendingPing(TEST_TYPE, {}, {});229 // Try to load it: there should be no error.230 yield TelemetryStorage.loadPendingPing(pendingPingId);231 let h = Telemetry.getHistogramById("TELEMETRY_PENDING_LOAD_FAILURE_READ").snapshot();232 Assert.equal(h.sum, 0, "Telemetry must not report a pending ping load failure");233 h = Telemetry.getHistogramById("TELEMETRY_PENDING_LOAD_FAILURE_PARSE").snapshot();234 Assert.equal(h.sum, 0, "Telemetry must not report a pending ping parse failure");235 // Delete it from the disk, so that its id will be kept in the cache but it will236 // fail loading the file.237 yield OS.File.remove(getSavePathForPingId(pendingPingId));238 // Try to load a pending ping which isn't there anymore.239 yield Assert.rejects(TelemetryStorage.loadPendingPing(pendingPingId),240 "Telemetry must fail loading a ping which isn't there");241 h = Telemetry.getHistogramById("TELEMETRY_PENDING_LOAD_FAILURE_READ").snapshot();242 Assert.equal(h.sum, 1, "Telemetry must report a pending ping load failure");243 h = Telemetry.getHistogramById("TELEMETRY_PENDING_LOAD_FAILURE_PARSE").snapshot();244 Assert.equal(h.sum, 0, "Telemetry must not report a pending ping parse failure");245 // Save a new ping, so that it gets in the pending pings cache.246 pendingPingId = yield TelemetryController.addPendingPing(TEST_TYPE, {}, {});247 // Overwrite it with a corrupted JSON file and then try to load it.248 const INVALID_JSON = "{ invalid,JSON { {1}";249 yield OS.File.writeAtomic(getSavePathForPingId(pendingPingId), INVALID_JSON, { encoding: "utf-8" });250 // Try to load the ping with the corrupted JSON content.251 yield Assert.rejects(TelemetryStorage.loadPendingPing(pendingPingId),252 "Telemetry must fail loading a corrupted ping");253 h = Telemetry.getHistogramById("TELEMETRY_PENDING_LOAD_FAILURE_READ").snapshot();254 Assert.equal(h.sum, 1, "Telemetry must report a pending ping load failure");255 h = Telemetry.getHistogramById("TELEMETRY_PENDING_LOAD_FAILURE_PARSE").snapshot();256 Assert.equal(h.sum, 1, "Telemetry must report a pending ping parse failure");257 let exists = yield OS.File.exists(getSavePathForPingId(pendingPingId));258 Assert.ok(!exists, "The unparseable ping should have been removed");259 yield TelemetryStorage.testClearPendingPings();260});261/**262 * Create some recent and overdue pings and verify that they get sent.263 */264add_task(function* test_overdue_pings_trigger_send() {265 let pingTypes = [266 { num: RECENT_PINGS },267 { num: OVERDUE_PINGS, age: OVERDUE_PING_FILE_AGE },268 ];269 let pings = yield createSavedPings(pingTypes);270 let recentPings = pings.slice(0, RECENT_PINGS);271 let overduePings = pings.slice(-OVERDUE_PINGS);272 yield TelemetryController.testReset();273 yield TelemetrySend.testWaitOnOutgoingPings();274 assertReceivedPings(TOTAL_EXPECTED_PINGS);275 yield assertNotSaved(recentPings);276 yield assertNotSaved(overduePings);277 Assert.equal(TelemetrySend.overduePingsCount, overduePings.length,278 "Should have tracked the correct amount of overdue pings");279 yield TelemetryStorage.testClearPendingPings();280});281/**282 * Create a ping in the old format, send it, and make sure the request URL contains283 * the correct version query parameter.284 */285add_task(function* test_overdue_old_format() {286 // A test ping in the old, standard format.287 const PING_OLD_FORMAT = {288 slug: "1234567abcd",289 reason: "test-ping",290 payload: {291 info: {292 reason: "test-ping",293 OS: "XPCShell",294 appID: "SomeId",295 appVersion: "1.0",296 appName: "XPCShell",297 appBuildID: "123456789",298 appUpdateChannel: "Test",299 platformBuildID: "987654321",300 },301 },302 };303 const filePath =304 Path.join(Constants.Path.profileDir, PING_SAVE_FOLDER, PING_OLD_FORMAT.slug);305 // Write the ping to file and make it overdue.306 yield TelemetryStorage.savePing(PING_OLD_FORMAT, true);307 yield File.setDates(filePath, null, Date.now() - OVERDUE_PING_FILE_AGE);308 let receivedPings = 0;309 // Register a new prefix handler to validate the URL.310 PingServer.registerPingHandler(request => {311 // Check that we have a version query parameter in the URL.312 Assert.notEqual(request.queryString, "");313 // Make sure the version in the query string matches the old ping format version.314 let params = request.queryString.split("&");315 Assert.ok(params.find(p => p == "v=1"));316 receivedPings++;317 });318 yield TelemetryController.testReset();319 yield TelemetrySend.testWaitOnOutgoingPings();320 Assert.equal(receivedPings, 1, "We must receive a ping in the old format.");321 yield TelemetryStorage.testClearPendingPings();322 PingServer.resetPingHandler();323});324add_task(function* test_pendingPingsQuota() {325 const PING_TYPE = "foo";326 // Disable upload so pings don't get sent and removed from the pending pings directory.327 Services.prefs.setBoolPref(PREF_FHR_UPLOAD, false);328 // Remove all the pending pings then startup and wait for the cleanup task to complete.329 // There should be nothing to remove.330 yield TelemetryStorage.testClearPendingPings();331 yield TelemetryController.testReset();332 yield TelemetrySend.testWaitOnOutgoingPings();333 yield TelemetryStorage.testPendingQuotaTaskPromise();334 // Remove the pending deletion ping generated when flipping FHR upload off.335 yield TelemetryStorage.testClearPendingPings();336 let expectedPrunedPings = [];337 let expectedNotPrunedPings = [];338 let checkPendingPings = Task.async(function*() {339 // Check that the pruned pings are not on disk anymore.340 for (let prunedPingId of expectedPrunedPings) {341 yield Assert.rejects(TelemetryStorage.loadPendingPing(prunedPingId),342 "Ping " + prunedPingId + " should have been pruned.");343 const pingPath = getSavePathForPingId(prunedPingId);344 Assert.ok(!(yield OS.File.exists(pingPath)), "The ping should not be on the disk anymore.");345 }346 // Check that the expected pings are there.347 for (let expectedPingId of expectedNotPrunedPings) {348 Assert.ok((yield TelemetryStorage.loadPendingPing(expectedPingId)),349 "Ping" + expectedPingId + " should be among the pending pings.");350 }351 });352 let pendingPingsInfo = [];353 let pingsSizeInBytes = 0;354 // Create 10 pings to test the pending pings quota.355 for (let days = 1; days < 11; days++) {356 const date = fakeNow(2010, 1, days, 1, 1, 0);357 const pingId = yield TelemetryController.addPendingPing(PING_TYPE, {}, {});358 // Find the size of the ping.359 const pingFilePath = getSavePathForPingId(pingId);360 const pingSize = (yield OS.File.stat(pingFilePath)).size;361 // Add the info at the beginning of the array, so that most recent pings come first.362 pendingPingsInfo.unshift({id: pingId, size: pingSize, timestamp: date.getTime() });363 // Set the last modification date.364 yield OS.File.setDates(pingFilePath, null, date.getTime());365 // Add it to the pending ping directory size.366 pingsSizeInBytes += pingSize;367 }368 // We need to test the pending pings size before we hit the quota, otherwise a special369 // value is recorded.370 Telemetry.getHistogramById("TELEMETRY_PENDING_PINGS_SIZE_MB").clear();371 Telemetry.getHistogramById("TELEMETRY_PENDING_PINGS_EVICTED_OVER_QUOTA").clear();372 Telemetry.getHistogramById("TELEMETRY_PENDING_EVICTING_OVER_QUOTA_MS").clear();373 yield TelemetryController.testReset();374 yield TelemetryStorage.testPendingQuotaTaskPromise();375 // Check that the correct values for quota probes are reported when no quota is hit.376 let h = Telemetry.getHistogramById("TELEMETRY_PENDING_PINGS_SIZE_MB").snapshot();377 Assert.equal(h.sum, Math.round(pingsSizeInBytes / 1024 / 1024),378 "Telemetry must report the correct pending pings directory size.");379 h = Telemetry.getHistogramById("TELEMETRY_PENDING_PINGS_EVICTED_OVER_QUOTA").snapshot();380 Assert.equal(h.sum, 0, "Telemetry must report 0 evictions if quota is not hit.");381 h = Telemetry.getHistogramById("TELEMETRY_PENDING_EVICTING_OVER_QUOTA_MS").snapshot();382 Assert.equal(h.sum, 0, "Telemetry must report a null elapsed time if quota is not hit.");383 // Set the quota to 80% of the space.384 const testQuotaInBytes = pingsSizeInBytes * 0.8;385 fakePendingPingsQuota(testQuotaInBytes);386 // The storage prunes pending pings until we reach 90% of the requested storage quota.387 // Based on that, find how many pings should be kept.388 const safeQuotaSize = Math.round(testQuotaInBytes * 0.9);389 let sizeInBytes = 0;390 let pingsWithinQuota = [];391 let pingsOutsideQuota = [];392 for (let pingInfo of pendingPingsInfo) {393 sizeInBytes += pingInfo.size;394 if (sizeInBytes >= safeQuotaSize) {395 pingsOutsideQuota.push(pingInfo.id);396 continue;397 }398 pingsWithinQuota.push(pingInfo.id);399 }400 expectedNotPrunedPings = pingsWithinQuota;401 expectedPrunedPings = pingsOutsideQuota;402 // Reset TelemetryController to start the pending pings cleanup.403 yield TelemetryController.testReset();404 yield TelemetryStorage.testPendingQuotaTaskPromise();405 yield checkPendingPings();406 h = Telemetry.getHistogramById("TELEMETRY_PENDING_PINGS_EVICTED_OVER_QUOTA").snapshot();407 Assert.equal(h.sum, pingsOutsideQuota.length,408 "Telemetry must correctly report the over quota pings evicted from the pending pings directory.");409 h = Telemetry.getHistogramById("TELEMETRY_PENDING_PINGS_SIZE_MB").snapshot();410 Assert.equal(h.sum, 17, "Pending pings quota was hit, a special size must be reported.");411 // Trigger a cleanup again and make sure we're not removing anything.412 yield TelemetryController.testReset();413 yield TelemetryStorage.testPendingQuotaTaskPromise();414 yield checkPendingPings();415 const OVERSIZED_PING_ID = "9b21ec8f-f762-4d28-a2c1-44e1c4694f24";416 // Create a pending oversized ping.417 const OVERSIZED_PING = {418 id: OVERSIZED_PING_ID,419 type: PING_TYPE,420 creationDate: (new Date()).toISOString(),421 // Generate a 2MB string to use as the ping payload.422 payload: generateRandomString(2 * 1024 * 1024),423 };424 yield TelemetryStorage.savePendingPing(OVERSIZED_PING);425 // Reset the histograms.426 Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_PENDING").clear();427 Telemetry.getHistogramById("TELEMETRY_DISCARDED_PENDING_PINGS_SIZE_MB").clear();428 // Try to manually load the oversized ping.429 yield Assert.rejects(TelemetryStorage.loadPendingPing(OVERSIZED_PING_ID),430 "The oversized ping should have been pruned.");431 Assert.ok(!(yield OS.File.exists(getSavePathForPingId(OVERSIZED_PING_ID))),432 "The ping should not be on the disk anymore.");433 // Make sure we're correctly updating the related histograms.434 h = Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_PENDING").snapshot();435 Assert.equal(h.sum, 1, "Telemetry must report 1 oversized ping in the pending pings directory.");436 h = Telemetry.getHistogramById("TELEMETRY_DISCARDED_PENDING_PINGS_SIZE_MB").snapshot();437 Assert.equal(h.counts[2], 1, "Telemetry must report a 2MB, oversized, ping.");438 // Save the ping again to check if it gets pruned when scanning the pings directory.439 yield TelemetryStorage.savePendingPing(OVERSIZED_PING);440 expectedPrunedPings.push(OVERSIZED_PING_ID);441 // Scan the pending pings directory.442 yield TelemetryController.testReset();443 yield TelemetryStorage.testPendingQuotaTaskPromise();444 yield checkPendingPings();445 // Make sure we're correctly updating the related histograms.446 h = Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_PENDING").snapshot();447 Assert.equal(h.sum, 2, "Telemetry must report 1 oversized ping in the pending pings directory.");448 h = Telemetry.getHistogramById("TELEMETRY_DISCARDED_PENDING_PINGS_SIZE_MB").snapshot();449 Assert.equal(h.counts[2], 2, "Telemetry must report two 2MB, oversized, pings.");450 Services.prefs.setBoolPref(PREF_FHR_UPLOAD, true);451});452add_task(function* teardown() {453 yield PingServer.stop();...

test_TelemetryController.js

Source:test_TelemetryController.js

1/* Any copyright is dedicated to the Public Domain.2 http://creativecommons.org/publicdomain/zero/1.0/3*/4/* This testcase triggers two telemetry pings.5 *6 * Telemetry code keeps histograms of past telemetry pings. The first7 * ping populates these histograms. One of those histograms is then8 * checked in the second request.9 */10Cu.import("resource://gre/modules/ClientID.jsm");11Cu.import("resource://gre/modules/Services.jsm");12Cu.import("resource://gre/modules/XPCOMUtils.jsm", this);13Cu.import("resource://gre/modules/TelemetryController.jsm", this);14Cu.import("resource://gre/modules/TelemetryStorage.jsm", this);15Cu.import("resource://gre/modules/TelemetrySend.jsm", this);16Cu.import("resource://gre/modules/TelemetryArchive.jsm", this);17Cu.import("resource://gre/modules/Task.jsm", this);18Cu.import("resource://gre/modules/Promise.jsm", this);19Cu.import("resource://gre/modules/Preferences.jsm");20const PING_FORMAT_VERSION = 4;21const DELETION_PING_TYPE = "deletion";22const TEST_PING_TYPE = "test-ping-type";23const PLATFORM_VERSION = "1.9.2";24const APP_VERSION = "1";25const APP_NAME = "XPCShell";26const PREF_BRANCH = "toolkit.telemetry.";27const PREF_ENABLED = PREF_BRANCH + "enabled";28const PREF_ARCHIVE_ENABLED = PREF_BRANCH + "archive.enabled";29const PREF_FHR_UPLOAD_ENABLED = "datareporting.healthreport.uploadEnabled";30const PREF_UNIFIED = PREF_BRANCH + "unified";31var gClientID = null;32function sendPing(aSendClientId, aSendEnvironment) {33 if (PingServer.started) {34 TelemetrySend.setServer("http://localhost:" + PingServer.port);35 } else {36 TelemetrySend.setServer("http://doesnotexist");37 }38 let options = {39 addClientId: aSendClientId,40 addEnvironment: aSendEnvironment,41 };42 return TelemetryController.submitExternalPing(TEST_PING_TYPE, {}, options);43}44function checkPingFormat(aPing, aType, aHasClientId, aHasEnvironment) {45 const MANDATORY_PING_FIELDS = [46 "type", "id", "creationDate", "version", "application", "payload"47 ];48 const APPLICATION_TEST_DATA = {49 buildId: gAppInfo.appBuildID,50 name: APP_NAME,51 version: APP_VERSION,52 displayVersion: AppConstants.MOZ_APP_VERSION_DISPLAY,53 vendor: "Mozilla",54 platformVersion: PLATFORM_VERSION,55 xpcomAbi: "noarch-spidermonkey",56 };57 // Check that the ping contains all the mandatory fields.58 for (let f of MANDATORY_PING_FIELDS) {59 Assert.ok(f in aPing, f + " must be available.");60 }61 Assert.equal(aPing.type, aType, "The ping must have the correct type.");62 Assert.equal(aPing.version, PING_FORMAT_VERSION, "The ping must have the correct version.");63 // Test the application section.64 for (let f in APPLICATION_TEST_DATA) {65 Assert.equal(aPing.application[f], APPLICATION_TEST_DATA[f],66 f + " must have the correct value.");67 }68 // We can't check the values for channel and architecture. Just make69 // sure they are in.70 Assert.ok("architecture" in aPing.application,71 "The application section must have an architecture field.");72 Assert.ok("channel" in aPing.application,73 "The application section must have a channel field.");74 // Check the clientId and environment fields, as needed.75 Assert.equal("clientId" in aPing, aHasClientId);76 Assert.equal("environment" in aPing, aHasEnvironment);77}78add_task(function* test_setup() {79 // Addon manager needs a profile directory80 do_get_profile();81 loadAddonManager("xpcshell@tests.mozilla.org", "XPCShell", "1", "1.9.2");82 // Make sure we don't generate unexpected pings due to pref changes.83 yield setEmptyPrefWatchlist();84 Services.prefs.setBoolPref(PREF_ENABLED, true);85 Services.prefs.setBoolPref(PREF_FHR_UPLOAD_ENABLED, true);86 yield new Promise(resolve =>87 Telemetry.asyncFetchTelemetryData(wrapWithExceptionHandler(resolve)));88});89add_task(function* asyncSetup() {90 yield TelemetryController.testSetup();91});92// Ensure that not overwriting an existing file fails silently93add_task(function* test_overwritePing() {94 let ping = {id: "foo"};95 yield TelemetryStorage.savePing(ping, true);96 yield TelemetryStorage.savePing(ping, false);97 yield TelemetryStorage.cleanupPingFile(ping);98});99// Checks that a sent ping is correctly received by a dummy http server.100add_task(function* test_simplePing() {101 PingServer.start();102 // Update the Telemetry Server preference with the address of the local server.103 // Otherwise we might end up sending stuff to a non-existing server after104 // |TelemetryController.testReset| is called.105 Preferences.set(TelemetryController.Constants.PREF_SERVER, "http://localhost:" + PingServer.port);106 yield sendPing(false, false);107 let request = yield PingServer.promiseNextRequest();108 // Check that we have a version query parameter in the URL.109 Assert.notEqual(request.queryString, "");110 // Make sure the version in the query string matches the new ping format version.111 let params = request.queryString.split("&");112 Assert.ok(params.find(p => p == ("v=" + PING_FORMAT_VERSION)));113 let ping = decodeRequestPayload(request);114 checkPingFormat(ping, TEST_PING_TYPE, false, false);115});116add_task(function* test_disableDataUpload() {117 const isUnified = Preferences.get(PREF_UNIFIED, false);118 if (!isUnified) {119 // Skipping the test if unified telemetry is off, as no deletion ping will120 // be generated.121 return;122 }123 // Disable FHR upload: this should trigger a deletion ping.124 Preferences.set(PREF_FHR_UPLOAD_ENABLED, false);125 let ping = yield PingServer.promiseNextPing();126 checkPingFormat(ping, DELETION_PING_TYPE, true, false);127 // Wait on ping activity to settle.128 yield TelemetrySend.testWaitOnOutgoingPings();129 // Restore FHR Upload.130 Preferences.set(PREF_FHR_UPLOAD_ENABLED, true);131 // Simulate a failure in sending the deletion ping by disabling the HTTP server.132 yield PingServer.stop();133 // Try to send a ping. It will be saved as pending and get deleted when disabling upload.134 TelemetryController.submitExternalPing(TEST_PING_TYPE, {});135 // Disable FHR upload to send a deletion ping again.136 Preferences.set(PREF_FHR_UPLOAD_ENABLED, false);137 // Wait on sending activity to settle, as |TelemetryController.testReset()| doesn't do that.138 yield TelemetrySend.testWaitOnOutgoingPings();139 // Wait for the pending pings to be deleted. Resetting TelemetryController doesn't140 // trigger the shutdown, so we need to call it ourselves.141 yield TelemetryStorage.shutdown();142 // Simulate a restart, and spin the send task.143 yield TelemetryController.testReset();144 // Disabling Telemetry upload must clear out all the pending pings.145 let pendingPings = yield TelemetryStorage.loadPendingPingList();146 Assert.equal(pendingPings.length, 1,147 "All the pending pings but the deletion ping should have been deleted");148 // Enable the ping server again.149 PingServer.start();150 // We set the new server using the pref, otherwise it would get reset with151 // |TelemetryController.testReset|.152 Preferences.set(TelemetryController.Constants.PREF_SERVER, "http://localhost:" + PingServer.port);153 // Stop the sending task and then start it again.154 yield TelemetrySend.shutdown();155 // Reset the controller to spin the ping sending task.156 yield TelemetryController.testReset();157 ping = yield PingServer.promiseNextPing();158 checkPingFormat(ping, DELETION_PING_TYPE, true, false);159 // Wait on ping activity to settle before moving on to the next test. If we were160 // to shut down telemetry, even though the PingServer caught the expected pings,161 // TelemetrySend could still be processing them (clearing pings would happen in162 // a couple of ticks). Shutting down would cancel the request and save them as163 // pending pings.164 yield TelemetrySend.testWaitOnOutgoingPings();165 // Restore FHR Upload.166 Preferences.set(PREF_FHR_UPLOAD_ENABLED, true);167});168add_task(function* test_pingHasClientId() {169 const PREF_CACHED_CLIENTID = "toolkit.telemetry.cachedClientID";170 // Make sure we have no cached client ID for this test: we'll try to send171 // a ping with it while Telemetry is being initialized.172 Preferences.reset(PREF_CACHED_CLIENTID);173 yield TelemetryController.testShutdown();174 yield ClientID._reset();175 yield TelemetryStorage.testClearPendingPings();176 // And also clear the counter histogram since we're here.177 let h = Telemetry.getHistogramById("TELEMETRY_PING_SUBMISSION_WAITING_CLIENTID");178 h.clear();179 // Init telemetry and try to send a ping with a client ID.180 let promisePingSetup = TelemetryController.testReset();181 yield sendPing(true, false);182 Assert.equal(h.snapshot().sum, 1,183 "We must have a ping waiting for the clientId early during startup.");184 // Wait until we are fully initialized. Pings will be assembled but won't get185 // sent before then.186 yield promisePingSetup;187 let ping = yield PingServer.promiseNextPing();188 // Fetch the client ID after initializing and fetching the the ping, so we189 // don't unintentionally trigger its loading. We'll still need the client ID190 // to see if the ping looks sane.191 gClientID = yield ClientID.getClientID();192 checkPingFormat(ping, TEST_PING_TYPE, true, false);193 Assert.equal(ping.clientId, gClientID, "The correct clientId must be reported.");194 // Shutdown Telemetry so we can safely restart it.195 yield TelemetryController.testShutdown();196 yield TelemetryStorage.testClearPendingPings();197 // We should have cached the client ID now. Lets confirm that by checking it before198 // the async ping setup is finished.199 h.clear();200 promisePingSetup = TelemetryController.testReset();201 yield sendPing(true, false);202 yield promisePingSetup;203 // Check that we received the cached client id.204 Assert.equal(h.snapshot().sum, 0, "We must have used the cached clientId.");205 ping = yield PingServer.promiseNextPing();206 checkPingFormat(ping, TEST_PING_TYPE, true, false);207 Assert.equal(ping.clientId, gClientID,208 "Telemetry should report the correct cached clientId.");209 // Check that sending a ping without relying on the cache, after the210 // initialization, still works.211 Preferences.reset(PREF_CACHED_CLIENTID);212 yield TelemetryController.testShutdown();213 yield TelemetryStorage.testClearPendingPings();214 yield TelemetryController.testReset();215 yield sendPing(true, false);216 ping = yield PingServer.promiseNextPing();217 checkPingFormat(ping, TEST_PING_TYPE, true, false);218 Assert.equal(ping.clientId, gClientID, "The correct clientId must be reported.");219 Assert.equal(h.snapshot().sum, 0, "No ping should have been waiting for a clientId.");220});221add_task(function* test_pingHasEnvironment() {222 // Send a ping with the environment data.223 yield sendPing(false, true);224 let ping = yield PingServer.promiseNextPing();225 checkPingFormat(ping, TEST_PING_TYPE, false, true);226 // Test a field in the environment build section.227 Assert.equal(ping.application.buildId, ping.environment.build.buildId);228});229add_task(function* test_pingHasEnvironmentAndClientId() {230 // Send a ping with the environment data and client id.231 yield sendPing(true, true);232 let ping = yield PingServer.promiseNextPing();233 checkPingFormat(ping, TEST_PING_TYPE, true, true);234 // Test a field in the environment build section.235 Assert.equal(ping.application.buildId, ping.environment.build.buildId);236 // Test that we have the correct clientId.237 Assert.equal(ping.clientId, gClientID, "The correct clientId must be reported.");238});239add_task(function* test_archivePings() {240 let now = new Date(2009, 10, 18, 12, 0, 0);241 fakeNow(now);242 // Disable ping upload so that pings don't get sent.243 // With unified telemetry the FHR upload pref controls this,244 // with non-unified telemetry the Telemetry enabled pref.245 const isUnified = Preferences.get(PREF_UNIFIED, false);246 const uploadPref = isUnified ? PREF_FHR_UPLOAD_ENABLED : PREF_ENABLED;247 Preferences.set(uploadPref, false);248 // If we're using unified telemetry, disabling ping upload will generate a "deletion"249 // ping. Catch it.250 if (isUnified) {251 let ping = yield PingServer.promiseNextPing();252 checkPingFormat(ping, DELETION_PING_TYPE, true, false);253 }254 // Register a new Ping Handler that asserts if a ping is received, then send a ping.255 PingServer.registerPingHandler(() => Assert.ok(false, "Telemetry must not send pings if not allowed to."));256 let pingId = yield sendPing(true, true);257 // Check that the ping was archived, even with upload disabled.258 let ping = yield TelemetryArchive.promiseArchivedPingById(pingId);259 Assert.equal(ping.id, pingId, "TelemetryController should still archive pings.");260 // Check that pings don't get archived if not allowed to.261 now = new Date(2010, 10, 18, 12, 0, 0);262 fakeNow(now);263 Preferences.set(PREF_ARCHIVE_ENABLED, false);264 pingId = yield sendPing(true, true);265 let promise = TelemetryArchive.promiseArchivedPingById(pingId);266 Assert.ok((yield promiseRejects(promise)),267 "TelemetryController should not archive pings if the archive pref is disabled.");268 // Enable archiving and the upload so that pings get sent and archived again.269 Preferences.set(uploadPref, true);270 Preferences.set(PREF_ARCHIVE_ENABLED, true);271 now = new Date(2014, 6, 18, 22, 0, 0);272 fakeNow(now);273 // Restore the non asserting ping handler.274 PingServer.resetPingHandler();275 pingId = yield sendPing(true, true);276 // Check that we archive pings when successfully sending them.277 yield PingServer.promiseNextPing();278 ping = yield TelemetryArchive.promiseArchivedPingById(pingId);279 Assert.equal(ping.id, pingId,280 "TelemetryController should still archive pings if ping upload is enabled.");281});282// Test that we fuzz the submission time around midnight properly283// to avoid overloading the telemetry servers.284add_task(function* test_midnightPingSendFuzzing() {285 const fuzzingDelay = 60 * 60 * 1000;286 fakeMidnightPingFuzzingDelay(fuzzingDelay);287 let now = new Date(2030, 5, 1, 11, 0, 0);288 fakeNow(now);289 let waitForTimer = () => new Promise(resolve => {290 fakePingSendTimer((callback, timeout) => {291 resolve([callback, timeout]);292 }, () => {});293 });294 PingServer.clearRequests();295 yield TelemetryController.testReset();296 // A ping after midnight within the fuzzing delay should not get sent.297 now = new Date(2030, 5, 2, 0, 40, 0);298 fakeNow(now);299 PingServer.registerPingHandler((req, res) => {300 Assert.ok(false, "No ping should be received yet.");301 });302 let timerPromise = waitForTimer();303 yield sendPing(true, true);304 let [timerCallback, timerTimeout] = yield timerPromise;305 Assert.ok(!!timerCallback);306 Assert.deepEqual(futureDate(now, timerTimeout), new Date(2030, 5, 2, 1, 0, 0));307 // A ping just before the end of the fuzzing delay should not get sent.308 now = new Date(2030, 5, 2, 0, 59, 59);309 fakeNow(now);310 timerPromise = waitForTimer();311 yield sendPing(true, true);312 [timerCallback, timerTimeout] = yield timerPromise;313 Assert.deepEqual(timerTimeout, 1 * 1000);314 // Restore the previous ping handler.315 PingServer.resetPingHandler();316 // Setting the clock to after the fuzzing delay, we should trigger the two ping sends317 // with the timer callback.318 now = futureDate(now, timerTimeout);319 fakeNow(now);320 yield timerCallback();321 const pings = yield PingServer.promiseNextPings(2);322 for (let ping of pings) {323 checkPingFormat(ping, TEST_PING_TYPE, true, true);324 }325 yield TelemetrySend.testWaitOnOutgoingPings();326 // Moving the clock further we should still send pings immediately.327 now = futureDate(now, 5 * 60 * 1000);328 yield sendPing(true, true);329 let ping = yield PingServer.promiseNextPing();330 checkPingFormat(ping, TEST_PING_TYPE, true, true);331 yield TelemetrySend.testWaitOnOutgoingPings();332 // Check that pings shortly before midnight are immediately sent.333 now = fakeNow(2030, 5, 3, 23, 59, 0);334 yield sendPing(true, true);335 ping = yield PingServer.promiseNextPing();336 checkPingFormat(ping, TEST_PING_TYPE, true, true);337 yield TelemetrySend.testWaitOnOutgoingPings();338 // Clean-up.339 fakeMidnightPingFuzzingDelay(0);340 fakePingSendTimer(() => {}, () => {});341});342add_task(function* test_changePingAfterSubmission() {343 // Submit a ping with a custom payload.344 let payload = { canary: "test" };345 let pingPromise = TelemetryController.submitExternalPing(TEST_PING_TYPE, payload, options);346 // Change the payload with a predefined value.347 payload.canary = "changed";348 // Wait for the ping to be archived.349 const pingId = yield pingPromise;350 // Make sure our changes didn't affect the submitted payload.351 let archivedCopy = yield TelemetryArchive.promiseArchivedPingById(pingId);352 Assert.equal(archivedCopy.payload.canary, "test",353 "The payload must not be changed after being submitted.");354});355add_task(function* test_telemetryEnabledUnexpectedValue() {356 // Remove the default value for toolkit.telemetry.enabled from the default prefs.357 // Otherwise, we wouldn't be able to set the pref to a string.358 let defaultPrefBranch = Services.prefs.getDefaultBranch(null);359 defaultPrefBranch.deleteBranch(PREF_ENABLED);360 // Set the preferences controlling the Telemetry status to a string.361 Preferences.set(PREF_ENABLED, "false");362 // Check that Telemetry is not enabled.363 yield TelemetryController.testReset();364 Assert.equal(Telemetry.canRecordExtended, false,365 "Invalid values must not enable Telemetry recording.");366 // Delete the pref again.367 defaultPrefBranch.deleteBranch(PREF_ENABLED);368 // Make sure that flipping it to true works.369 Preferences.set(PREF_ENABLED, true);370 yield TelemetryController.testReset();371 Assert.equal(Telemetry.canRecordExtended, true,372 "True must enable Telemetry recording.");373 // Also check that the false works as well.374 Preferences.set(PREF_ENABLED, false);375 yield TelemetryController.testReset();376 Assert.equal(Telemetry.canRecordExtended, false,377 "False must disable Telemetry recording.");378});379add_task(function* test_telemetryCleanFHRDatabase() {380 const FHR_DBNAME_PREF = "datareporting.healthreport.dbName";381 const CUSTOM_DB_NAME = "unlikely.to.be.used.sqlite";382 const DEFAULT_DB_NAME = "healthreport.sqlite";383 // Check that we're able to remove a FHR DB with a custom name.384 const CUSTOM_DB_PATHS = [385 OS.Path.join(OS.Constants.Path.profileDir, CUSTOM_DB_NAME),386 OS.Path.join(OS.Constants.Path.profileDir, CUSTOM_DB_NAME + "-wal"),387 OS.Path.join(OS.Constants.Path.profileDir, CUSTOM_DB_NAME + "-shm"),388 ];389 Preferences.set(FHR_DBNAME_PREF, CUSTOM_DB_NAME);390 // Write fake DB files to the profile directory.391 for (let dbFilePath of CUSTOM_DB_PATHS) {392 yield OS.File.writeAtomic(dbFilePath, "some data");393 }394 // Trigger the cleanup and check that the files were removed.395 yield TelemetryStorage.removeFHRDatabase();396 for (let dbFilePath of CUSTOM_DB_PATHS) {397 Assert.ok(!(yield OS.File.exists(dbFilePath)), "The DB must not be on the disk anymore: " + dbFilePath);398 }399 // We should not break anything if there's no DB file.400 yield TelemetryStorage.removeFHRDatabase();401 // Check that we're able to remove a FHR DB with the default name.402 Preferences.reset(FHR_DBNAME_PREF);403 const DEFAULT_DB_PATHS = [404 OS.Path.join(OS.Constants.Path.profileDir, DEFAULT_DB_NAME),405 OS.Path.join(OS.Constants.Path.profileDir, DEFAULT_DB_NAME + "-wal"),406 OS.Path.join(OS.Constants.Path.profileDir, DEFAULT_DB_NAME + "-shm"),407 ];408 // Write fake DB files to the profile directory.409 for (let dbFilePath of DEFAULT_DB_PATHS) {410 yield OS.File.writeAtomic(dbFilePath, "some data");411 }412 // Trigger the cleanup and check that the files were removed.413 yield TelemetryStorage.removeFHRDatabase();414 for (let dbFilePath of DEFAULT_DB_PATHS) {415 Assert.ok(!(yield OS.File.exists(dbFilePath)), "The DB must not be on the disk anymore: " + dbFilePath);416 }417});418add_task(function* stopServer() {419 yield PingServer.stop();...

test_tcping.py

Source:test_tcping.py

...15 tcp_ping = ping.TCPing(destination=args.destination,16 port=args.port,17 timeout=args.timeout,18 use_ipv6=args.use_ipv6)19 tcp_ping.do_ping()20 self.assertEqual(len(tcp_ping.measures), 1)21 def test_ping_domain_incorrect(self):22 with mock.patch('socket.socket') as mock_socket:23 mock_socket.return_value.recv.return_value = b""24 cmd_parser = tcping.create_cmd_parser()25 args = cmd_parser.parse_args(['google.csom'])26 tcp_ping = ping.TCPing(destination=args.destination,27 port=args.port,28 timeout=args.timeout,29 use_ipv6=args.use_ipv6)30 with self.assertRaises(errors.InvalidIpOrDomain):31 tcp_ping.do_ping()32 def test_ping_ip_standart(self):33 with mock.patch('socket.socket') as mock_socket:34 mock_socket.return_value.recv.return_value = b""35 cmd_parser = tcping.create_cmd_parser()36 args = cmd_parser.parse_args(['64.233.165.101', '-c', '10'])37 tcp_ping = ping.TCPing(destination=args.destination,38 port=args.port,39 timeout=args.timeout,40 use_ipv6=args.use_ipv6)41 tcp_ping.do_ping()42 self.assertEqual(len(tcp_ping.measures), 1)43 def test_ping_ip_incorrect(self):44 with mock.patch('socket.socket') as mock_socket:45 mock_socket.return_value.recv.return_value = b""46 cmd_parser = tcping.create_cmd_parser()47 args = cmd_parser.parse_args(['64.233.165.101.123.214'])48 tcp_ping = ping.TCPing(destination=args.destination,49 port=args.port,50 timeout=args.timeout,51 use_ipv6=args.use_ipv6)52 with self.assertRaises(errors.InvalidIpOrDomain):53 tcp_ping.do_ping()54class TestParsing(unittest.TestCase):55 def test_parsing_with_args(self):56 cmd_parser = tcping.create_cmd_parser()57 args = cmd_parser.parse_args(['google.com',58 '-t', '10',59 '-p', '443',60 '-6'])61 tcp_ping = ping.TCPing(destination=args.destination, port=args.port,62 timeout=args.timeout, use_ipv6=args.use_ipv6)63 self.assertEqual(tcp_ping.use_ipv6, True)64 self.assertEqual(tcp_ping.timeout, 10)65 self.assertEqual(tcp_ping.port, 443)66 def test_parsing_without_args_from_cmd(self):67 cmd_parser = tcping.create_cmd_parser()...

Show more

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.

LambdaTest Learning Hubs:

• JUnit Tutorial
• TestNG Tutorial
• Webdriver Tutorial
• WebDriverIO Tutorial
• Protractor Tutorial
• Selenium 4 Tutorial
• Jenkins Tutorial
• NUnit Tutorial
• Jest Tutorial
• Playwright Tutorial
• Cypress Tutorial
• PyTest Tutorial

YouTube

You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.

Run localstack automation tests on LambdaTest cloud grid

Perform automation testing on 3000+ real desktop and mobile devices online.

View sample repo View DocumentationSign up Free

›

_