KaneAI - World’s First E2E Software Testing Agent.Book a Demo Join TestMu Conference

How to use links method in Kiwi

Best Python code snippet using Kiwi_python

test_NewTabUtils.js

Source:test_NewTabUtils.js

...29  );30  // isTopSiteGivenProvider() and getProviderLinks() should still return results31  // even when cache is empty or being populated.32  Assert.ok(!NewTabUtils.isTopSiteGivenProvider("example1.com", provider));33  do_check_links(NewTabUtils.getProviderLinks(provider), []);34  await promise;35  // Once the cache is populated, we get the expected results36  Assert.ok(NewTabUtils.isTopSiteGivenProvider("example1.com", provider));37  do_check_links(NewTabUtils.getProviderLinks(provider), expectedLinks);38  NewTabUtils.links.removeProvider(provider);39});40add_task(async function notifyLinkDelete() {41  let expectedLinks = makeLinks(0, 3, 1);42  let provider = new TestProvider(done => done(expectedLinks));43  provider.maxNumLinks = expectedLinks.length;44  NewTabUtils.initWithoutProviders();45  NewTabUtils.links.addProvider(provider);46  await new Promise(resolve => NewTabUtils.links.populateCache(resolve));47  do_check_links(NewTabUtils.links.getLinks(), expectedLinks);48  // Remove a link.49  let removedLink = expectedLinks[2];50  provider.notifyLinkChanged(removedLink, 2, true);51  let links = NewTabUtils.links._providers.get(provider);52  // Check that sortedLinks is correctly updated.53  do_check_links(NewTabUtils.links.getLinks(), expectedLinks.slice(0, 2));54  // Check that linkMap is accurately updated.55  Assert.equal(links.linkMap.size, 2);56  Assert.ok(links.linkMap.get(expectedLinks[0].url));57  Assert.ok(links.linkMap.get(expectedLinks[1].url));58  Assert.ok(!links.linkMap.get(removedLink.url));59  // Check that siteMap is correctly updated.60  Assert.equal(links.siteMap.size, 2);61  Assert.ok(links.siteMap.has(NewTabUtils.extractSite(expectedLinks[0].url)));62  Assert.ok(links.siteMap.has(NewTabUtils.extractSite(expectedLinks[1].url)));63  Assert.ok(!links.siteMap.has(NewTabUtils.extractSite(removedLink.url)));64  NewTabUtils.links.removeProvider(provider);65});66add_task(async function populatePromise() {67  let count = 0;68  let expectedLinks = makeLinks(0, 10, 2);69  let getLinksFcn = async function(callback) {70    // Should not be calling getLinksFcn twice71    count++;72    Assert.equal(count, 1);73    await Promise.resolve();74    callback(expectedLinks);75  };76  let provider = new TestProvider(getLinksFcn);77  NewTabUtils.initWithoutProviders();78  NewTabUtils.links.addProvider(provider);79  NewTabUtils.links.populateProviderCache(provider, () => {});80  NewTabUtils.links.populateProviderCache(provider, () => {81    do_check_links(NewTabUtils.links.getLinks(), expectedLinks);82    NewTabUtils.links.removeProvider(provider);83  });84});85add_task(async function isTopSiteGivenProvider() {86  let expectedLinks = makeLinks(0, 10, 2);87  // The lowest 2 frecencies have the same base domain.88  expectedLinks[expectedLinks.length - 2].url =89    expectedLinks[expectedLinks.length - 1].url + "Test";90  let provider = new TestProvider(done => done(expectedLinks));91  provider.maxNumLinks = expectedLinks.length;92  NewTabUtils.initWithoutProviders();93  NewTabUtils.links.addProvider(provider);94  await new Promise(resolve => NewTabUtils.links.populateCache(resolve));95  Assert.equal(96    NewTabUtils.isTopSiteGivenProvider("example2.com", provider),97    true98  );99  Assert.equal(100    NewTabUtils.isTopSiteGivenProvider("example1.com", provider),101    false102  );103  // Push out frecency 2 because the maxNumLinks is reached when adding frecency 3104  let newLink = makeLink(3);105  provider.notifyLinkChanged(newLink);106  // There is still a frecent url with example2 domain, so it's still frecent.107  Assert.equal(108    NewTabUtils.isTopSiteGivenProvider("example3.com", provider),109    true110  );111  Assert.equal(112    NewTabUtils.isTopSiteGivenProvider("example2.com", provider),113    true114  );115  // Push out frecency 3116  newLink = makeLink(5);117  provider.notifyLinkChanged(newLink);118  // Push out frecency 4119  newLink = makeLink(9);120  provider.notifyLinkChanged(newLink);121  // Our count reached 0 for the example2.com domain so it's no longer a frecent site.122  Assert.equal(123    NewTabUtils.isTopSiteGivenProvider("example5.com", provider),124    true125  );126  Assert.equal(127    NewTabUtils.isTopSiteGivenProvider("example2.com", provider),128    false129  );130  NewTabUtils.links.removeProvider(provider);131});132add_task(async function multipleProviders() {133  // Make each provider generate NewTabUtils.links.maxNumLinks links to check134  // that no more than maxNumLinks are actually returned in the merged list.135  let evenLinks = makeLinks(0, 2 * NewTabUtils.links.maxNumLinks, 2);136  let evenProvider = new TestProvider(done => done(evenLinks));137  let oddLinks = makeLinks(0, 2 * NewTabUtils.links.maxNumLinks - 1, 2);138  let oddProvider = new TestProvider(done => done(oddLinks));139  NewTabUtils.initWithoutProviders();140  NewTabUtils.links.addProvider(evenProvider);141  NewTabUtils.links.addProvider(oddProvider);142  await new Promise(resolve => NewTabUtils.links.populateCache(resolve));143  let links = NewTabUtils.links.getLinks();144  let expectedLinks = makeLinks(145    NewTabUtils.links.maxNumLinks,146    2 * NewTabUtils.links.maxNumLinks,147    1148  );149  Assert.equal(links.length, NewTabUtils.links.maxNumLinks);150  do_check_links(links, expectedLinks);151  NewTabUtils.links.removeProvider(evenProvider);152  NewTabUtils.links.removeProvider(oddProvider);153});154add_task(async function changeLinks() {155  let expectedLinks = makeLinks(0, 20, 2);156  let provider = new TestProvider(done => done(expectedLinks));157  NewTabUtils.initWithoutProviders();158  NewTabUtils.links.addProvider(provider);159  await new Promise(resolve => NewTabUtils.links.populateCache(resolve));160  do_check_links(NewTabUtils.links.getLinks(), expectedLinks);161  // Notify of a new link.162  let newLink = makeLink(19);163  expectedLinks.splice(1, 0, newLink);164  provider.notifyLinkChanged(newLink);165  do_check_links(NewTabUtils.links.getLinks(), expectedLinks);166  // Notify of a link that's changed sort criteria.167  newLink.frecency = 17;168  expectedLinks.splice(1, 1);169  expectedLinks.splice(2, 0, newLink);170  provider.notifyLinkChanged({171    url: newLink.url,172    frecency: 17,173  });174  do_check_links(NewTabUtils.links.getLinks(), expectedLinks);175  // Notify of a link that's changed title.176  newLink.title = "My frecency is now 17";177  provider.notifyLinkChanged({178    url: newLink.url,179    title: newLink.title,180  });181  do_check_links(NewTabUtils.links.getLinks(), expectedLinks);182  // Notify of a new link again, but this time make it overflow maxNumLinks.183  provider.maxNumLinks = expectedLinks.length;184  newLink = makeLink(21);185  expectedLinks.unshift(newLink);186  expectedLinks.pop();187  Assert.equal(expectedLinks.length, provider.maxNumLinks); // Sanity check.188  provider.notifyLinkChanged(newLink);189  do_check_links(NewTabUtils.links.getLinks(), expectedLinks);190  // Notify of many links changed.191  expectedLinks = makeLinks(0, 3, 1);192  provider.notifyManyLinksChanged();193  // Since _populateProviderCache() is async, we must wait until the provider's194  // populate promise has been resolved.195  await NewTabUtils.links._providers.get(provider).populatePromise;196  // NewTabUtils.links will now repopulate its cache197  do_check_links(NewTabUtils.links.getLinks(), expectedLinks);198  NewTabUtils.links.removeProvider(provider);199});200add_task(async function oneProviderAlreadyCached() {201  let links1 = makeLinks(0, 10, 1);202  let provider1 = new TestProvider(done => done(links1));203  NewTabUtils.initWithoutProviders();204  NewTabUtils.links.addProvider(provider1);205  await new Promise(resolve => NewTabUtils.links.populateCache(resolve));206  do_check_links(NewTabUtils.links.getLinks(), links1);207  let links2 = makeLinks(10, 20, 1);208  let provider2 = new TestProvider(done => done(links2));209  NewTabUtils.links.addProvider(provider2);210  await new Promise(resolve => NewTabUtils.links.populateCache(resolve));211  do_check_links(NewTabUtils.links.getLinks(), links2.concat(links1));212  NewTabUtils.links.removeProvider(provider1);213  NewTabUtils.links.removeProvider(provider2);214});215add_task(async function newLowRankedLink() {216  // Init a provider with 10 links and make its maximum number also 10.217  let links = makeLinks(0, 10, 1);218  let provider = new TestProvider(done => done(links));219  provider.maxNumLinks = links.length;220  NewTabUtils.initWithoutProviders();221  NewTabUtils.links.addProvider(provider);222  await new Promise(resolve => NewTabUtils.links.populateCache(resolve));223  do_check_links(NewTabUtils.links.getLinks(), links);224  // Notify of a new link that's low-ranked enough not to make the list.225  let newLink = makeLink(0);226  provider.notifyLinkChanged(newLink);227  do_check_links(NewTabUtils.links.getLinks(), links);228  // Notify about the new link's title change.229  provider.notifyLinkChanged({230    url: newLink.url,231    title: "a new title",232  });233  do_check_links(NewTabUtils.links.getLinks(), links);234  NewTabUtils.links.removeProvider(provider);235});236add_task(async function extractSite() {237  // All these should extract to the same site238  [239    "mozilla.org",240    "m.mozilla.org",241    "mobile.mozilla.org",242    "www.mozilla.org",243    "www3.mozilla.org",244  ].forEach(host => {245    let url = "http://" + host;246    Assert.equal(247      NewTabUtils.extractSite(url),...

cleanfailed.js

Source:cleanfailed.js

1const fs = require('fs')2const storage = require('../../util/storage.js')3const removeRss = require('../../util/removeRss.js')4const channelTracker = require('../../util/channelTracker.js')5const fileOps = require('../../util/fileOps.js')6exports.normal = function (bot, message) {7  const currentGuilds = storage.currentGuilds8  const failedLinks = storage.failedLinks9  let content = message.content.split(' ')10  content.shift()11  const reason = content.join(' ').trim()12  const affectedGuilds = []13  const links = []14  currentGuilds.forEach(function (guildRss, guildId) {15    const rssList = guildRss.sources16    for (var failedLink in failedLinks) {17      if (typeof failedLinks[failedLink] !== 'string' || failedLinks[failedLink] < 100) continue18      for (var rssName in rssList) {19        if (rssList[rssName].link === failedLink) {20          if (!links.includes(rssList[rssName].link)) links.push(rssList[rssName].link)21          if (!affectedGuilds.includes(guildId)) affectedGuilds.push(guildId)22        }23      }24    }25  })26  if (links.length === 0) {27    let cleaned = false28    if (!bot.shard) {29      for (var j in failedLinks) {30        if (typeof failedLinks[j] === 'string' || failedLinks[j] >= 100) {31          cleaned = true32          delete failedLinks[j]33        }34      }35    }36    fs.writeFileSync('./settings/failedLinks.json', JSON.stringify(failedLinks, null, 2))37    return message.channel.send(cleaned ? `No links were eligible to be removed from guilds, but outdated links have been deleted from failedLinks.json.` : `No links are eligible to be cleaned out from failedLinks.json.`).catch(err => console.log(`Promise Warning: forceremove 1.`, err.message || err))38  }39  let msg = '```'40  for (var x in links) {41    msg += `\n${links[x]}`42  }43  msg += '```'44  const collector = message.channel.createMessageCollector(m => m.author.id === message.author.id, {time: 240000})45  channelTracker.add(message.channel.id)46  message.channel.send(`The list of links (${links.length}) to be cleaned out from failedLinks.json ${reason ? 'and the reason for removal ' : ''}is shown below.\n\n${reason ? '```Reason: ' + reason + '```\n' : ''}${msg.length > 1950 ? '```Unable to print links to discord - exceeds 1950 chars. Please see console.```' : msg}\n\nDo you want to continue? Type **Yes** to confirm, or **No** to cancel.`)47  .then(function (prompt) {48    if (msg.length > 1950) {49      console.log(`Bot Controller: Links that are about to be forcibly removed for cleanup as requested by (${message.author.id}, ${message.author.username}): `)50      for (var a in links) {51        console.info(`\n${links[a]}`)52      }53    }54    collector.on('collect', function (m) {55      if (m.content !== 'Yes' && m.content !== 'No') return message.channel.send('That is not a valid Option. Please type either **Yes** or **No**.')56      collector.stop()57      if (m.content === 'No') return message.channel.send(`Force removal canceled.`)58      let removedLinks = []59      for (var i in affectedGuilds) {60        const guildRss = currentGuilds.get(affectedGuilds[i])61        const rssList = guildRss.sources62        let names = []63        for (var name in rssList) {64          for (var e in links) {65            if (rssList[name].link === links[e]) {66              removedLinks.push(links[e])67              names.push(name)68            }69          }70        }71        for (var l in names) {72          const rssName = names[l]73          const link = rssList[rssName].link74          const channel = bot.channels.get(rssList[rssName].channel)75          if (reason && channel) channel.send(`**ATTENTION:** Feeds with link <${link}> have been forcibly removed from all servers. Reason: ${reason}`).catch(err => console.log(`Could not send force removal notification to server ${channel.guild.id}. `, err.message || err))76          removeRss(channel.guild.id, rssName)77          delete failedLinks[link]78          delete rssList[rssName]79        }80        fileOps.updateFile(affectedGuilds[i], guildRss)81      }82      if (removedLinks.length === 0) return message.channel.send('Unable to remove any links.').catch(err => console.log(`Promise Warning: forceremove 2. `, err.message || err))83      msg = '```'84      for (var p in removedLinks) {85        msg += `\n${removedLinks[p]}`86      }87      message.channel.send(`Successfully removed \`${removedLinks.length}\` source(s) from guilds. ${msg.length > 1950 ? '' : 'Links:```\n' + removedLinks + '```'}`)88      console.log(`Bot Controller: The following links have been forcibly removed by (${message.author.id}, ${message.author.username}): \n`, removedLinks)89      for (var j in failedLinks) {90        if (typeof failedLinks[j] === 'string' || failedLinks[j] === 100) delete failedLinks[j]91      }92      try { fs.writeFileSync('./settings/failedLinks.json', JSON.stringify(failedLinks, null, 2)) } catch (e) { console.log(`Bot Controller: Unable to write to failedLinks.json after cleanfailed. `, e.message || e) }93    })94    collector.on('end', function (collected, reason) {95      channelTracker.remove(message.channel.id)96      if (reason === 'time') return message.channel.send(`I have closed the menu due to inactivity.`).catch(err => console.log(`Promise Warning: Unable to send expired menu message.`, err.message || err))97      else if (reason !== 'user') return message.channel.send(reason)98    })99  }).catch(err => console.log(`Bot Controller: Could not send a list of links that are to be affected. `, err.message || err))100}101exports.sharded = function (bot, message, Manager) {102  const failedLinks = storage.failedLinks103  let content = message.content.split(' ')104  content.shift()105  const reason = content.join(' ').trim()106  const affectedGuilds = []107  const links = []108  bot.shard.broadcastEval(`109    const appDir = require('path').dirname(require.main.filename);110    const storage = require(appDir + '/util/storage.js');111    const currentGuilds = storage.currentGuilds;112    const failedLinks = storage.failedLinks;113    const obj = {affectedGuilds: [], links: []};114    const affectedGuilds = obj.affectedGuilds;115    const links = obj.links;116    currentGuilds.forEach(function (guildRss, guildId) {117      const rssList = guildRss.sources118      for (var failedLink in failedLinks) {119        if (typeof failedLinks[failedLink] !== 'string' || failedLinks[failedLink] < 100) continue120        for (var rssName in rssList) {121          if (rssList[rssName].link === failedLink) {122            if (!links.includes(rssList[rssName].link)) links.push(rssList[rssName].link);123            if (!affectedGuilds.includes(guildId)) affectedGuilds.push(guildId);124          }125        }126      }127    })128    obj;129  `).then(results => {130    for (var x in results) {131      const shardLinks = results[x].links132      const shardAffectedGuilds = results[x].affectedGuilds133      for (var a in shardLinks) if (!links.includes(shardLinks[a])) links.push(shardLinks[a])134      for (var b in shardAffectedGuilds) if (!affectedGuilds.includes(shardAffectedGuilds[b])) affectedGuilds.push(shardAffectedGuilds[b])135    }136    if (links.length === 0) {137      let cleaned = false138      for (var j in failedLinks) {139        if (typeof failedLinks[j] === 'string' || failedLinks[j] >= 100) {140          cleaned = true141          delete failedLinks[j]142        }143      }144      if (!cleaned) return message.channel.send(`No links are eligible to be cleaned out from failedLinks.json.`)145      try { fs.writeFileSync('./settings/failedLinks.json', JSON.stringify(failedLinks, null, 2)) } catch (e) { console.log(`Bot Controller: Unable to write to failedLinks after eval cleanfailed resulting in 0 eligible links. `, e.message || e) }146      return bot.shard.broadcastEval(`147        require(require('path').dirname(require.main.filename) + '/util/storage.js').failedLinks = JSON.parse('${JSON.stringify(failedLinks)}');148      `).then(() => {149        return message.channel.send(`No links were eligible to be removed from guilds, but outdated links have been deleted from failedLinks.json.`).catch(err => console.log(`Promise Warning: forceremove 1.`, err.message || err))150      }).catch(err => {151        console.log(`Bot Controller: Unable to broadcast failedLinks unification after eval cleanfailed resulting in 0 eligible links. `, err.message || err)152        message.channel.send(`Unable to broadcast failedLinks unification after eval cleanfailed resulting in 0 eligible links. `, err.message || err)153      })154    }155    let msg = '```'156    for (var u in links) msg += `\n${links[u]}`157    msg += '```'158    const collector = message.channel.createMessageCollector(m => m.author.id === message.author.id, {time: 240000})159    channelTracker.add(message.channel.id)160    message.channel.send(`The list of links (${links.length}) to be cleaned out from failedLinks.json ${reason ? 'and the reason for removal ' : ''}is shown below.\n\n${reason ? '```Reason: ' + reason + '```\n' : ''}${msg.length > 1950 ? '```Unable to print links to discord - exceeds 1950 chars. Please see console.```' : msg}\n\nDo you want to continue? Type **Yes** to confirm, or **No** to cancel.`)161    .then(function (prompt) {162      if (msg.length > 1950) {163        console.log(`Bot Controller: Links that are about to be forcibly removed for cleanup as requested by (${message.author.id}, ${message.author.username}): `)164        for (var a in links) {165          console.info(`\n${links[a]}`)166        }167      }168      collector.on('collect', function (m) {169        if (m.content !== 'Yes' && m.content !== 'No') return message.channel.send('That is not a valid Option. Please type either **Yes** or **No**.')170        collector.stop()171        if (m.content === 'No') return message.channel.send(`Force removal canceled.`)172        bot.shard.broadcastEval(`173          const appDir = require('path').dirname(require.main.filename);174          const storage = require(appDir + '/util/storage.js');175          const currentGuilds = storage.currentGuilds;176          const failedLinks = storage.failedLinks;177          const removeRss = require(appDir + '/util/removeRss.js');178          const fileOps = require(appDir + '/util/fileOps.js');179          const removedLinks = [];180          const affectedGuilds = JSON.parse('${JSON.stringify(affectedGuilds)}');181          const links = JSON.parse('${JSON.stringify(links)}')182          for (var i in affectedGuilds) {183            const guildRss = currentGuilds.get(affectedGuilds[i]);184            if (!guildRss) continue;185            const rssList = guildRss.sources;186            let names = [];187            for (var name in rssList) {188              for (var e in links) {189                if (rssList[name].link === links[e]) {190                  removedLinks.push(links[e]);191                  names.push(name);192                }193              }194            }195            for (var l in names) {196              const rssName = names[l];197              const link = rssList[rssName].link;198              const channel = this.channels.get(rssList[rssName].channel);199              if (${reason ? '"' + reason + '"' : undefined} && channel) channel.send('**ATTENTION:** Feeds with link <' + link + '> have been forcibly removed from all servers. Reason: ${reason ? '"' + reason + '"' : undefined}').catch(err => console.log('Could not send force removal notification to server ' + channel.guild.id + '. ', err.message || err))200              removeRss(channel.guild.id, rssName);201              delete failedLinks[link];202              delete rssList[rssName];203            }204            fileOps.updateFile(affectedGuilds[i], guildRss);205          }206          removedLinks;207        `).then(results => {208          let removed = []209          let msg = '```'210          let total = 0211          for (var a in results) {212            const removedLinks = results[a]213            if (!removedLinks) continue214            removed = removed.concat(removedLinks)215            total += removedLinks.length216          }217          for (var b in removed) {218            const link = removed[b]219            msg += '\n' + link220            delete failedLinks[link]221          }222          if (removed.length === 0) return message.channel.send('Unable to remove any links.').catch(err => console.log(`Promise Warning: forceremove 2s. `, err.message || err))223          for (var j in failedLinks) {224            if (typeof failedLinks[j] === 'string' || failedLinks[j] === 100) delete failedLinks[j]225          }226          bot.shard.broadcastEval(`227            require(require('path').dirname(require.main.filename) + '/util/storage.js').failedLinks = JSON.parse('${JSON.stringify(failedLinks)}');228          `).then(() => {229            message.channel.send(`Successfully removed \`${total}\` source(s) from guilds. ${msg.length > 1950 ? '' : 'Links:```\n' + removed + '```'}`)230            console.log(`Bot Controller: The following links have been forcibly removed by (${message.author.id}, ${message.author.username}): \n`, removed)231            try { fs.writeFileSync('./settings/failedLinks.json', JSON.stringify(failedLinks, null, 2)) } catch (e) { console.log(`Bot Controller: Unable to write to failedLinks.json after cleanfailed. `, e.message || e) }232          }).catch(err => console.log(`Bot Controller: Unable to broadcast failedLinks cache update after successful eval cleanfailed and removal action. `, err.message || err))233        }).catch(err => console.log(`Bot Controller: Unable to broadcast removal actions after successful eval cleanfailed. `, err.message || err))234      })235      collector.on('end', function (collected, reason) {236        channelTracker.remove(message.channel.id)237        if (reason === 'time') return message.channel.send(`I have closed the menu due to inactivity.`).catch(err => console.log(`Promise Warning: Unable to send expired menu message.`, err.message || err))238        else if (reason !== 'user') return message.channel.send(reason)239      })240    })241  }).catch(err => console.log(`Bot Controller: Unable to broadcast eval cleanfailed. `, err.message || err))...

LinkSaveIn.py

Source:LinkSaveIn.py

1# -*- coding: utf-8 -*-23#4# v2.01 - hagg5# * cnl2 and web links are skipped if JS is not available (instead of failing the package)6# * only best available link source is used (priority: cnl2>rsdf>ccf>dlc>web7#89from Crypto.Cipher import AES10from module.plugins.Crypter import Crypter11from module.unescape import unescape12import base6413import binascii14import re1516class LinkSaveIn(Crypter):17    __name__ = "LinkSaveIn"18    __type__ = "crypter"19    __pattern__ = r"http://(www\.)?linksave.in/(?P<id>\w+)$"20    __version__ = "2.01"21    __description__ = """LinkSave.in Crypter Plugin"""22    __author_name__ = ("fragonib")23    __author_mail__ = ("fragonib[AT]yahoo[DOT]es")2425    # Constants26    _JK_KEY_ = "jk"27    _CRYPTED_KEY_ = "crypted"28    HOSTER_DOMAIN = "linksave.in"29        30    def setup(self):31        self.html = None32        self.fileid = None33        self.captcha = False34        self.package = None35        self.preferred_sources = ['cnl2', 'rsdf', 'ccf', 'dlc', 'web']36        37    def decrypt(self, pyfile):3839        # Init40        self.package = pyfile.package()41        self.fileid = re.match(self.__pattern__, pyfile.url).group('id')42        self.req.cj.setCookie(self.HOSTER_DOMAIN, "Linksave_Language", "english")43        44        # Request package45        self.html = self.load(self.pyfile.url)46        if not self.isOnline():47            self.offline()48        49        # Check for protection    50        if self.isPasswordProtected():51            self.unlockPasswordProtection()52            self.handleErrors()53            54        if self.isCaptchaProtected():55            self.captcha = True56            self.unlockCaptchaProtection()57            self.handleErrors()5859        # Get package name and folder60        (package_name, folder_name) = self.getPackageInfo()6162        # Extract package links63        package_links = []64        for type_ in self.preferred_sources:65            package_links.extend(self.handleLinkSource(type_))66            if package_links:  # use only first source which provides links67                break68        package_links = set(package_links)6970        # Pack71        if package_links:72            self.packages = [(package_name, package_links, folder_name)]73        else:74            self.fail('Could not extract any links')7576    def isOnline(self):77        if "<big>Error 404 - Folder not found!</big>" in self.html:78            self.logDebug("File not found")79            return False80        return True81    82    def isPasswordProtected(self):83        if re.search(r'''<input.*?type="password"''', self.html):84            self.logDebug("Links are password protected")85            return True86        87    def isCaptchaProtected(self):88        if "<b>Captcha:</b>" in self.html:89            self.logDebug("Links are captcha protected")90            return True91        return False92        93    def unlockPasswordProtection(self):94        password = self.getPassword()95        self.logDebug("Submitting password [%s] for protected links" % password)96        post = {"id": self.fileid, "besucherpasswort": password, 'login': 'submit'}97        self.html = self.load(self.pyfile.url, post=post)98            99    def unlockCaptchaProtection(self):100        hash = re.search(r'name="hash" value="([^"]+)', self.html).group(1)101        captchaUrl = re.search(r'src=".(/captcha/cap.php\?hsh=[^"]+)', self.html).group(1)102        code = self.decryptCaptcha("http://linksave.in" + captchaUrl, forceUser=True)103        self.html = self.load(self.pyfile.url, post={"id": self.fileid, "hash": hash, "code": code})   104105    def getPackageInfo(self):106        name = self.pyfile.package().name107        folder = self.pyfile.package().folder108        self.logDebug("Defaulting to pyfile name [%s] and folder [%s] for package" % (name, folder))109        return name, folder110    111    def handleErrors(self):      112        if "The visitorpassword you have entered is wrong" in self.html:113            self.logDebug("Incorrect password, please set right password on 'Edit package' form and retry")114            self.fail("Incorrect password, please set right password on 'Edit package' form and retry")  115116        if self.captcha:          117            if "Wrong code. Please retry" in self.html:118                self.logDebug("Invalid captcha, retrying")119                self.invalidCaptcha()120                self.retry()121            else:122                self.correctCaptcha()123           124    def handleLinkSource(self, type_):125        if type_ == 'cnl2':126            return self.handleCNL2()127        elif type_ in ('rsdf', 'ccf', 'dlc'):128            return self.handleContainer(type_)129        elif type_ == 'web':130            return self.handleWebLinks()131        else:132            self.fail('unknown source type "%s" (this is probably a bug)' % type_)133134    def handleWebLinks(self):135        package_links = []136        self.logDebug("Search for Web links")137        if not self.js:138            self.logDebug("no JS -> skip Web links")139        else: 140            #@TODO: Gather paginated web links  141            pattern = r'<a href="http://linksave\.in/(\w{43})"'142            ids = re.findall(pattern, self.html)143            self.logDebug("Decrypting %d Web links" % len(ids))144            for i, id in enumerate(ids):145                try:146                    webLink = "http://linksave.in/%s" % id147                    self.logDebug("Decrypting Web link %d, %s" % (i+1, webLink))148                    fwLink = "http://linksave.in/fw-%s" % id149                    response = self.load(fwLink)150                    jscode = re.findall(r'<script type="text/javascript">(.*)</script>', response)[-1]151                    jseval = self.js.eval("document = { write: function(e) { return e; } }; %s" % jscode)152                    dlLink = re.search(r'http://linksave\.in/dl-\w+', jseval).group(0)153                    self.logDebug("JsEngine returns value [%s] for redirection link" % dlLink)154                    response = self.load(dlLink)155                    link = unescape(re.search(r'<iframe src="(.+?)"', response).group(1))156                    package_links.append(link)157                except Exception, detail:158                    self.logDebug("Error decrypting Web link %s, %s" % (webLink, detail))    159        return package_links160    161    def handleContainer(self, type_):162        package_links = []163        type_ = type_.lower()164        self.logDebug('Seach for %s Container links' % type_.upper())165        if not type_.isalnum():  # check to prevent broken re-pattern (cnl2,rsdf,ccf,dlc,web are all alpha-numeric)166            self.fail('unknown container type "%s" (this is probably a bug)' % type_)167        pattern = r"\('%s_link'\).href=unescape\('(.*?\.%s)'\)" % (type_, type_)168        containersLinks = re.findall(pattern, self.html)169        self.logDebug("Found %d %s Container links" % (len(containersLinks), type_.upper()))170        for containerLink in containersLinks:171            link = "http://linksave.in/%s" % unescape(containerLink)172            package_links.append(link)173        return package_links174175    def handleCNL2(self):176        package_links = []177        self.logDebug("Search for CNL2 links")178        if not self.js:179            self.logDebug("no JS -> skip CNL2 links")180        elif 'cnl2_load' in self.html:181            try:182                (vcrypted, vjk) = self._getCipherParams()183                for (crypted, jk) in zip(vcrypted, vjk):184                    package_links.extend(self._getLinks(crypted, jk))185            except:186                self.fail("Unable to decrypt CNL2 links")            187        return package_links188    189    def _getCipherParams(self):190            191        # Get jk192        jk_re = r'<INPUT.*?NAME="%s".*?VALUE="(.*?)"' % LinkSaveIn._JK_KEY_       193        vjk = re.findall(jk_re, self.html)194        195        # Get crypted196        crypted_re = r'<INPUT.*?NAME="%s".*?VALUE="(.*?)"' % LinkSaveIn._CRYPTED_KEY_       197        vcrypted = re.findall(crypted_re, self.html)198199        # Log and return200        self.logDebug("Detected %d crypted blocks" % len(vcrypted))201        return vcrypted, vjk202203    def _getLinks(self, crypted, jk):204205        # Get key206        jreturn = self.js.eval("%s f()" % jk)207        self.logDebug("JsEngine returns value [%s]" % jreturn)208        key = binascii.unhexlify(jreturn)209210        # Decode crypted211        crypted = base64.standard_b64decode(crypted)212213        # Decrypt214        Key = key215        IV = key216        obj = AES.new(Key, AES.MODE_CBC, IV)217        text = obj.decrypt(crypted)218219        # Extract links220        text = text.replace("\x00", "").replace("\r", "")221        links = text.split("\n")222        links = filter(lambda x: x != "", links)223224        # Log and return225        self.logDebug("Package has %d links" % len(links))226        return links
...

NCryptIn.py

Source:NCryptIn.py

1# -*- coding: utf-8 -*-23from Crypto.Cipher import AES4from module.plugins.Crypter import Crypter5from module.plugins.ReCaptcha import ReCaptcha6import base647import binascii8import re910class NCryptIn(Crypter):11    __name__ = "NCryptIn"12    __type__ = "crypter"13    __pattern__ = r"http://(?:www\.)?ncrypt.in/folder-([^/\?]+)"14    __version__ = "1.21"15    __description__ = """NCrypt.in Crypter Plugin"""16    __author_name__ = ("fragonib")17    __author_mail__ = ("fragonib[AT]yahoo[DOT]es")1819    # Constants20    _JK_KEY_ = "jk"21    _CRYPTED_KEY_ = "crypted"22        23    def setup(self):24        self.html = None25        self.cleanedHtml = None26        self.captcha = False27        self.package = None2829    def decrypt(self, pyfile):30        31        # Init32        self.package = pyfile.package()33        34        # Request package35        self.html = self.load(self.pyfile.url)36        self.cleanedHtml = self.removeCrap(self.html)37        if not self.isOnline():38            self.offline()39        40        # Check for protection    41        if self.isProtected():42            self.html = self.unlockProtection()43            self.cleanedHtml = self.removeCrap(self.html)44            self.handleErrors()4546        # Get package name and folder47        (package_name, folder_name) = self.getPackageInfo()4849        # Extract package links50        package_links = []51        package_links.extend(self.handleWebLinks())52        package_links.extend(self.handleContainers())53        package_links.extend(self.handleCNL2())54        package_links = set(package_links)5556        # Pack57        self.packages = [(package_name, package_links, folder_name)]58        59    def removeCrap(self, content):60        patterns = (r'(type="hidden".*?(name=".*?")?.*?value=".*?")',61                    r'display:none;">(.*?)</(div|span)>',62                    r'<div\s+class="jdownloader"(.*?)</div>',63                    r'<iframe\s+style="display:none(.*?)</iframe>')64        for pattern in patterns:65            rexpr = re.compile(pattern, re.DOTALL)66            content = re.sub(rexpr, "", content)67        return content68        69    def isOnline(self):        70        if "Your folder does not exist" in self.cleanedHtml:71            self.logDebug("File not found")72            return False73        return True74    75    def isProtected(self):76        if re.search(r'''<form.*?name.*?protected.*?>''', self.cleanedHtml):77            self.logDebug("Links are protected")78            return True79        return False80    81    def getPackageInfo(self):82        title_re = r'<h2><span.*?class="arrow".*?>(?P<title>[^<]+).*?</span>.*?</h2>'83        m = re.findall(title_re, self.html, re.DOTALL)84        if m is not None:85            title = m[-1].strip()86            name = folder = title87            self.logDebug("Found name [%s] and folder [%s] in package info" % (name, folder))88        else:89            name = self.package.name90            folder = self.package.folder91            self.logDebug("Package info not found, defaulting to pyfile name [%s] and folder [%s]" % (name, folder))92        return name, folder93    94    def unlockProtection(self):95        96        postData = {}97                98        form = re.search(r'''<form\ name="protected"(.*?)</form>''', self.cleanedHtml, re.DOTALL).group(1)99        100        # Submit package password101        if "password" in form:102            password = self.getPassword()103            self.logDebug("Submitting password [%s] for protected links" % password)104            postData['password'] = password105        106        # Resolve anicaptcha107        if "anicaptcha" in form:108            self.captcha = True109            self.logDebug("Captcha protected, resolving captcha")110            captchaUri = re.search(r'src="(/temp/anicaptcha/[^"]+)', form).group(1)111            captcha = self.decryptCaptcha("http://ncrypt.in" + captchaUri)112            self.logDebug("Captcha resolved [%s]" % captcha)113            postData['captcha'] = captcha114        115        # Resolve recaptcha           116        if "recaptcha" in form:117            self.captcha = True    118            id = re.search(r'\?k=(.*?)"', form).group(1)119            self.logDebug("Resolving ReCaptcha with key [%s]" % id)120            recaptcha = ReCaptcha(self)121            challenge, code = recaptcha.challenge(id)122            postData['recaptcha_challenge_field'] = challenge123            postData['recaptcha_response_field'] = code124                   125        # Unlock protection126        postData['submit_protected'] = 'Continue to folder '127        return self.load(self.pyfile.url, post=postData)128        129    def handleErrors(self):130                   131        if "This password is invalid!" in self.cleanedHtml:132            self.logDebug("Incorrect password, please set right password on 'Edit package' form and retry")133            self.fail("Incorrect password, please set right password on 'Edit package' form and retry")  134135        if self.captcha:          136            if "The securitycheck was wrong!" in self.cleanedHtml:137                self.logDebug("Invalid captcha, retrying")138                self.invalidCaptcha()139                self.retry()140            else:141                self.correctCaptcha()142143    def handleWebLinks(self):144        package_links = []145        self.logDebug("Handling Web links")146        147        pattern = r"(http://ncrypt\.in/link-.*?=)"148        links = re.findall(pattern, self.html)149        self.logDebug("Decrypting %d Web links" % len(links))150        for i, link in enumerate(links):151            self.logDebug("Decrypting Web link %d, %s" % (i+1, link))152            try:153                url = link.replace("link-", "frame-")154                link = self.load(url, just_header=True)['location']155                package_links.append(link)156            except Exception, detail:157                self.logDebug("Error decrypting Web link %s, %s" % (link, detail))    158        return package_links159    160    def handleContainers(self):161        package_links = []162        self.logDebug("Handling Container links")163        164        pattern = r"/container/(rsdf|dlc|ccf)/([a-z0-9]+)"165        containersLinks = re.findall(pattern, self.html)166        self.logDebug("Decrypting %d Container links" % len(containersLinks))167        for containerLink in containersLinks:168            link = "http://ncrypt.in/container/%s/%s.%s" % (containerLink[0], containerLink[1], containerLink[0])169            package_links.append(link)170        return package_links171                172    def handleCNL2(self):173        package_links = []174        self.logDebug("Handling CNL2 links")175        176        if 'cnl2_output' in self.cleanedHtml:177            try:178                (vcrypted, vjk) = self._getCipherParams()179                for (crypted, jk) in zip(vcrypted, vjk):180                    package_links.extend(self._getLinks(crypted, jk))181            except:182                self.fail("Unable to decrypt CNL2 links")            183        return package_links184    185    def _getCipherParams(self):186            187        pattern = r'<input.*?name="%s".*?value="(.*?)"'    188            189        # Get jk190        jk_re = pattern % NCryptIn._JK_KEY_       191        vjk = re.findall(jk_re, self.html)192        193        # Get crypted194        crypted_re = pattern % NCryptIn._CRYPTED_KEY_      195        vcrypted = re.findall(crypted_re, self.html)196197        # Log and return198        self.logDebug("Detected %d crypted blocks" % len(vcrypted))199        return vcrypted, vjk200201    def _getLinks(self, crypted, jk):202203        # Get key204        jreturn = self.js.eval("%s f()" % jk)205        self.logDebug("JsEngine returns value [%s]" % jreturn)206        key = binascii.unhexlify(jreturn)207208        # Decode crypted209        crypted = base64.standard_b64decode(crypted)210211        # Decrypt212        Key = key213        IV = key214        obj = AES.new(Key, AES.MODE_CBC, IV)215        text = obj.decrypt(crypted)216217        # Extract links218        text = text.replace("\x00", "").replace("\r", "")219        links = text.split("\n")220        links = filter(lambda x: x != "", links)221222        # Log and return223        self.logDebug("Block has %d links" % len(links))
...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.