How to use tokenizer class

Best Atoum code snippet using tokenizer

ParserTest.php

Source:ParserTest.php Github

copy

Full Screen

1<?php2/*3 * This file is part of Mustache.php.4 *5 * (c) 2010-2017 Justin Hileman6 *7 * For the full copyright and license information, please view the LICENSE8 * file that was distributed with this source code.9 */10/**11 * @group unit12 */13class Mustache_Test_ParserTest extends PHPUnit_Framework_TestCase14{15 /**16 * @dataProvider getTokenSets17 */18 public function testParse($tokens, $expected)19 {20 $parser = new Mustache_Parser();21 $this->assertEquals($expected, $parser->parse($tokens));22 }23 public function getTokenSets()24 {25 return array(26 array(27 array(),28 array(),29 ),30 array(31 array(array(32 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,33 Mustache_Tokenizer::LINE => 0,34 Mustache_Tokenizer::VALUE => 'text',35 )),36 array(array(37 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,38 Mustache_Tokenizer::LINE => 0,39 Mustache_Tokenizer::VALUE => 'text',40 )),41 ),42 array(43 array(array(44 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED,45 Mustache_Tokenizer::LINE => 0,46 Mustache_Tokenizer::NAME => 'name',47 )),48 array(array(49 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED,50 Mustache_Tokenizer::LINE => 0,51 Mustache_Tokenizer::NAME => 'name',52 )),53 ),54 array(55 array(56 array(57 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,58 Mustache_Tokenizer::LINE => 0,59 Mustache_Tokenizer::VALUE => 'foo',60 ),61 array(62 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_INVERTED,63 Mustache_Tokenizer::LINE => 0,64 Mustache_Tokenizer::INDEX => 123,65 Mustache_Tokenizer::NAME => 'parent',66 ),67 array(68 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED,69 Mustache_Tokenizer::LINE => 0,70 Mustache_Tokenizer::NAME => 'name',71 ),72 array(73 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_END_SECTION,74 Mustache_Tokenizer::LINE => 0,75 Mustache_Tokenizer::INDEX => 456,76 Mustache_Tokenizer::NAME => 'parent',77 ),78 array(79 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,80 Mustache_Tokenizer::LINE => 0,81 Mustache_Tokenizer::VALUE => 'bar',82 ),83 ),84 array(85 array(86 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,87 Mustache_Tokenizer::LINE => 0,88 Mustache_Tokenizer::VALUE => 'foo',89 ),90 array(91 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_INVERTED,92 Mustache_Tokenizer::NAME => 'parent',93 Mustache_Tokenizer::LINE => 0,94 Mustache_Tokenizer::INDEX => 123,95 Mustache_Tokenizer::END => 456,96 Mustache_Tokenizer::NODES => array(97 array(98 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED,99 Mustache_Tokenizer::LINE => 0,100 Mustache_Tokenizer::NAME => 'name',101 ),102 ),103 ),104 array(105 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,106 Mustache_Tokenizer::LINE => 0,107 Mustache_Tokenizer::VALUE => 'bar',108 ),109 ),110 ),111 // This *would* be an invalid inheritance parse tree, but that pragma112 // isn't enabled so it'll thunk it back into an "escaped" token:113 array(114 array(115 array(116 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_BLOCK_VAR,117 Mustache_Tokenizer::NAME => 'foo',118 Mustache_Tokenizer::OTAG => '{{',119 Mustache_Tokenizer::CTAG => '}}',120 Mustache_Tokenizer::LINE => 0,121 ),122 array(123 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,124 Mustache_Tokenizer::LINE => 0,125 Mustache_Tokenizer::VALUE => 'bar',126 ),127 ),128 array(129 array(130 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED,131 Mustache_Tokenizer::NAME => '$foo',132 Mustache_Tokenizer::OTAG => '{{',133 Mustache_Tokenizer::CTAG => '}}',134 Mustache_Tokenizer::LINE => 0,135 ),136 array(137 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,138 Mustache_Tokenizer::LINE => 0,139 Mustache_Tokenizer::VALUE => 'bar',140 ),141 ),142 ),143 array(144 array(145 array(146 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,147 Mustache_Tokenizer::LINE => 0,148 Mustache_Tokenizer::VALUE => ' ',149 ),150 array(151 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_DELIM_CHANGE,152 Mustache_Tokenizer::LINE => 0,153 ),154 array(155 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,156 Mustache_Tokenizer::LINE => 0,157 Mustache_Tokenizer::VALUE => " \n",158 ),159 array(160 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED,161 Mustache_Tokenizer::NAME => 'foo',162 Mustache_Tokenizer::OTAG => '[[',163 Mustache_Tokenizer::CTAG => ']]',164 Mustache_Tokenizer::LINE => 1,165 ),166 ),167 array(168 array(169 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED,170 Mustache_Tokenizer::NAME => 'foo',171 Mustache_Tokenizer::OTAG => '[[',172 Mustache_Tokenizer::CTAG => ']]',173 Mustache_Tokenizer::LINE => 1,174 ),175 ),176 ),177 );178 }179 /**180 * @dataProvider getInheritanceTokenSets181 */182 public function testParseWithInheritance($tokens, $expected)183 {184 $parser = new Mustache_Parser();185 $parser->setPragmas(array(Mustache_Engine::PRAGMA_BLOCKS));186 $this->assertEquals($expected, $parser->parse($tokens));187 }188 public function getInheritanceTokenSets()189 {190 return array(191 array(192 array(193 array(194 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_PARENT,195 Mustache_Tokenizer::NAME => 'foo',196 Mustache_Tokenizer::OTAG => '{{',197 Mustache_Tokenizer::CTAG => '}}',198 Mustache_Tokenizer::LINE => 0,199 Mustache_Tokenizer::INDEX => 8,200 ),201 array(202 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_BLOCK_VAR,203 Mustache_Tokenizer::NAME => 'bar',204 Mustache_Tokenizer::OTAG => '{{',205 Mustache_Tokenizer::CTAG => '}}',206 Mustache_Tokenizer::LINE => 0,207 Mustache_Tokenizer::INDEX => 16,208 ),209 array(210 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,211 Mustache_Tokenizer::LINE => 0,212 Mustache_Tokenizer::VALUE => 'baz',213 ),214 array(215 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_END_SECTION,216 Mustache_Tokenizer::NAME => 'bar',217 Mustache_Tokenizer::OTAG => '{{',218 Mustache_Tokenizer::CTAG => '}}',219 Mustache_Tokenizer::LINE => 0,220 Mustache_Tokenizer::INDEX => 19,221 ),222 array(223 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_END_SECTION,224 Mustache_Tokenizer::NAME => 'foo',225 Mustache_Tokenizer::OTAG => '{{',226 Mustache_Tokenizer::CTAG => '}}',227 Mustache_Tokenizer::LINE => 0,228 Mustache_Tokenizer::INDEX => 27,229 ),230 ),231 array(232 array(233 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_PARENT,234 Mustache_Tokenizer::NAME => 'foo',235 Mustache_Tokenizer::OTAG => '{{',236 Mustache_Tokenizer::CTAG => '}}',237 Mustache_Tokenizer::LINE => 0,238 Mustache_Tokenizer::INDEX => 8,239 Mustache_Tokenizer::END => 27,240 Mustache_Tokenizer::NODES => array(241 array(242 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_BLOCK_ARG,243 Mustache_Tokenizer::NAME => 'bar',244 Mustache_Tokenizer::OTAG => '{{',245 Mustache_Tokenizer::CTAG => '}}',246 Mustache_Tokenizer::LINE => 0,247 Mustache_Tokenizer::INDEX => 16,248 Mustache_Tokenizer::END => 19,249 Mustache_Tokenizer::NODES => array(250 array(251 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,252 Mustache_Tokenizer::LINE => 0,253 Mustache_Tokenizer::VALUE => 'baz',254 ),255 ),256 ),257 ),258 ),259 ),260 ),261 array(262 array(263 array(264 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_BLOCK_VAR,265 Mustache_Tokenizer::NAME => 'foo',266 Mustache_Tokenizer::OTAG => '{{',267 Mustache_Tokenizer::CTAG => '}}',268 Mustache_Tokenizer::LINE => 0,269 ),270 array(271 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,272 Mustache_Tokenizer::LINE => 0,273 Mustache_Tokenizer::VALUE => 'bar',274 ),275 array(276 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_END_SECTION,277 Mustache_Tokenizer::NAME => 'foo',278 Mustache_Tokenizer::OTAG => '{{',279 Mustache_Tokenizer::CTAG => '}}',280 Mustache_Tokenizer::LINE => 0,281 Mustache_Tokenizer::INDEX => 11,282 ),283 ),284 array(285 array(286 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_BLOCK_VAR,287 Mustache_Tokenizer::NAME => 'foo',288 Mustache_Tokenizer::OTAG => '{{',289 Mustache_Tokenizer::CTAG => '}}',290 Mustache_Tokenizer::LINE => 0,291 Mustache_Tokenizer::END => 11,292 Mustache_Tokenizer::NODES => array(293 array(294 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,295 Mustache_Tokenizer::LINE => 0,296 Mustache_Tokenizer::VALUE => 'bar',297 ),298 ),299 ),300 ),301 ),302 );303 }304 /**305 * @dataProvider getBadParseTrees306 * @expectedException Mustache_Exception_SyntaxException307 */308 public function testParserThrowsExceptions($tokens)309 {310 $parser = new Mustache_Parser();311 $parser->parse($tokens);312 }313 public function getBadParseTrees()314 {315 return array(316 // no close317 array(318 array(319 array(320 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_SECTION,321 Mustache_Tokenizer::NAME => 'parent',322 Mustache_Tokenizer::LINE => 0,323 Mustache_Tokenizer::INDEX => 123,324 ),325 ),326 ),327 // no close inverted328 array(329 array(330 array(331 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_INVERTED,332 Mustache_Tokenizer::NAME => 'parent',333 Mustache_Tokenizer::LINE => 0,334 Mustache_Tokenizer::INDEX => 123,335 ),336 ),337 ),338 // no opening inverted339 array(340 array(341 array(342 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_END_SECTION,343 Mustache_Tokenizer::NAME => 'parent',344 Mustache_Tokenizer::LINE => 0,345 Mustache_Tokenizer::INDEX => 123,346 ),347 ),348 ),349 // weird nesting350 array(351 array(352 array(353 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_SECTION,354 Mustache_Tokenizer::NAME => 'parent',355 Mustache_Tokenizer::LINE => 0,356 Mustache_Tokenizer::INDEX => 123,357 ),358 array(359 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_SECTION,360 Mustache_Tokenizer::NAME => 'child',361 Mustache_Tokenizer::LINE => 0,362 Mustache_Tokenizer::INDEX => 123,363 ),364 array(365 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_END_SECTION,366 Mustache_Tokenizer::NAME => 'parent',367 Mustache_Tokenizer::LINE => 0,368 Mustache_Tokenizer::INDEX => 123,369 ),370 array(371 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_END_SECTION,372 Mustache_Tokenizer::NAME => 'child',373 Mustache_Tokenizer::LINE => 0,374 Mustache_Tokenizer::INDEX => 123,375 ),376 ),377 ),378 // This *would* be a valid inheritance parse tree, but that pragma379 // isn't enabled here so it's going to fail :)380 array(381 array(382 array(383 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_BLOCK_VAR,384 Mustache_Tokenizer::NAME => 'foo',385 Mustache_Tokenizer::OTAG => '{{',386 Mustache_Tokenizer::CTAG => '}}',387 Mustache_Tokenizer::LINE => 0,388 ),389 array(390 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,391 Mustache_Tokenizer::LINE => 0,392 Mustache_Tokenizer::VALUE => 'bar',393 ),394 array(395 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_END_SECTION,396 Mustache_Tokenizer::NAME => 'foo',397 Mustache_Tokenizer::OTAG => '{{',398 Mustache_Tokenizer::CTAG => '}}',399 Mustache_Tokenizer::LINE => 0,400 Mustache_Tokenizer::INDEX => 11,401 ),402 ),403 ),404 );405 }406}...

Full Screen

Full Screen

schema.php

Source:schema.php Github

copy

Full Screen

...52 * @throws NotSupportedException53 */54 public function createFromString($str, $delimiter)55 {56 $tokenizer = Tokenizer::createFromString($str);57 foreach ($this->splitStatements($tokenizer, $delimiter) as $statement)58 {59 $this->executeStatement($statement);60 }61 }62 /**63 * Splits tokens array into bunch of individual DDL statements.64 *65 * @param Tokenizer $tokenizer Tokens container.66 * @param string $delimiter How to split DDL into statements.67 *68 * @return array[Tokenizer]69 */70 protected function splitStatements(Tokenizer $tokenizer, $delimiter = ';')71 {72 $result = array();73 $index = 0;74 $result[$index] = array();75 /** @var Token $prevToken */76 $prevToken = null;77 /** @var Token $token */78 foreach ($tokenizer->getTokens() as $token)79 {80 if (81 $token->text === $delimiter82 && $prevToken83 && strpos($prevToken->text, "\n") !== false84 )85 {86 $index++;87 $result[$index] = array();88 }89 elseif (90 strpos($token->text , "\n") !== false91 && $prevToken92 && $prevToken->text === $delimiter93 )94 {95 array_pop($result[$index]);96 $index++;97 $result[$index] = array();98 }99 else100 {101 $result[$index][] = $token;102 }103 $prevToken = $token;104 }105 foreach ($result as $i => $tokens)106 {107 $result[$i] = Tokenizer::createFromTokens($tokens);108 }109 return $result;110 }111 /**112 * Fills some schema part with information from one DDL statement.113 *114 * @param Tokenizer $tokenizer Statement tokens.115 *116 * @return void117 * @throws NotSupportedException118 */119 protected function executeStatement(Tokenizer $tokenizer)120 {121 /** @var Table $table */122 $tokenizer->resetState();123 $tokenizer->skipWhiteSpace();124 if ($tokenizer->testUpperText('CREATE'))125 {126 $this->executeCreate($tokenizer);127 }128 elseif ($tokenizer->testUpperText('INSERT'))129 {130 //skip insert into131 }132 elseif ($tokenizer->testUpperText('SET'))133 {134 //skip set identity_insert135 }136 elseif ($tokenizer->testUpperText('ALTER'))137 {138 $this->executeAlter($tokenizer);139 }140 elseif ($tokenizer->testUpperText('IF'))141 {142 $tokenizer->skipWhiteSpace();143 if ($tokenizer->testUpperText('OBJECT_ID'))144 {145 while (!$tokenizer->endOfInput())146 {147 if ($tokenizer->nextToken()->upper === 'CREATE')148 break;149 }150 $tokenizer->nextToken();151 $tokenizer->skipWhiteSpace();152 if ($tokenizer->testUpperText('TABLE'))153 {154 $this->executeCreateTable($tokenizer);155 }156 else157 {158 throw new NotSupportedException("'CREATE TABLE' expected. line:".$tokenizer->getCurrentToken()->line);159 }160 }161 elseif ($tokenizer->testUpperText('NOT'))162 {163 $tokenizer->skipWhiteSpace();164 if ($tokenizer->testUpperText('EXISTS'))165 {166 while (!$tokenizer->endOfInput())167 {168 if ($tokenizer->nextToken()->upper === 'CREATE')169 break;170 }171 $tokenizer->nextToken();172 $tokenizer->skipWhiteSpace();173 if ($tokenizer->testUpperText('UNIQUE'))174 {175 $unique = true;176 $tokenizer->skipWhiteSpace();177 }178 else179 {180 $unique = false;181 }182 if ($tokenizer->testUpperText('INDEX'))183 {184 $this->executeCreateIndex($tokenizer, $unique);185 }186 else187 {188 throw new NotSupportedException("'CREATE INDEX' expected. line:".$tokenizer->getCurrentToken()->line);189 }190 }191 else192 {193 throw new NotSupportedException("'NOT EXISTS' expected. line:".$tokenizer->getCurrentToken()->line);194 }195 }196 else197 {198 throw new NotSupportedException("'OBJECT_ID' expected. line:".$tokenizer->getCurrentToken()->line);199 }200 }201 elseif (!$tokenizer->endOfInput())202 {203 throw new NotSupportedException("'CREATE' expected. line:".$tokenizer->getCurrentToken()->line);204 }205 }206 /**207 * @param Tokenizer $tokenizer Statement tokens.208 *209 * @return void210 * @throws NotSupportedException211 */212 protected function executeCreate(Tokenizer $tokenizer)213 {214 $tokenizer->skipWhiteSpace();215 if ($tokenizer->testUpperText("OR"))216 {217 $tokenizer->skipWhiteSpace();218 if ($tokenizer->testUpperText("REPLACE"))219 $tokenizer->skipWhiteSpace();220 else221 throw new NotSupportedException("'OR REPLACE' expected. line:".$tokenizer->getCurrentToken()->line);222 }223 if ($tokenizer->testUpperText('TABLE'))224 {225 $this->executeCreateTable($tokenizer);226 }227 elseif ($tokenizer->testUpperText('INDEX'))228 {229 $this->executeCreateIndex($tokenizer, false);230 }231 elseif ($tokenizer->testUpperText('UNIQUE'))232 {233 $tokenizer->skipWhiteSpace();234 if ($tokenizer->testUpperText('INDEX'))235 $tokenizer->skipWhiteSpace();236 $this->executeCreateIndex($tokenizer, true);237 }238 elseif ($tokenizer->testUpperText('TRIGGER'))239 {240 $this->executeCreateTrigger($tokenizer);241 }242 elseif (243 $tokenizer->testUpperText('PROCEDURE')244 || $tokenizer->testUpperText('FUNCTION')245 || $tokenizer->testUpperText('TYPE')246 )247 {248 $this->executeCreateProcedure($tokenizer);249 }250 elseif ($tokenizer->testUpperText('SEQUENCE'))251 {252 $this->executeCreateSequence($tokenizer);253 }254 else255 {256 throw new NotSupportedException("TABLE|INDEX|UNIQUE|TRIGGER|PROCEDURE|FUNCTION|TYPE|SEQUENCE expected. line:".$tokenizer->getCurrentToken()->line);257 }258 }259 /**260 * @param Tokenizer $tokenizer Statement tokens.261 *262 * @return void263 * @throws NotSupportedException264 */265 protected function executeAlter(Tokenizer $tokenizer)266 {267 $tokenizer->skipWhiteSpace();268 if ($tokenizer->testUpperText('TABLE'))269 {270 $tokenizer->skipWhiteSpace();271 $tableName = $tokenizer->getCurrentToken()->text;272 /** @var Table $table */273 $table = $this->tables->search($tableName);274 if (!$table)275 {276 throw new NotSupportedException("Table [$tableName] not found. line: ".$tokenizer->getCurrentToken()->line);277 }278 $tokenizer->nextToken();279 $tokenizer->skipWhiteSpace();280 if ($tokenizer->testUpperText('ADD'))281 {282 $tokenizer->skipWhiteSpace();283 if ($tokenizer->testUpperText('CONSTRAINT'))284 {285 $tokenizer->skipWhiteSpace();286 $table->createConstraint($tokenizer);287 }288 }289 elseif ($tokenizer->testUpperText('NOCHECK') || $tokenizer->testUpperText('CHECK'))290 {291 //(NOCHECK|CHECK) CONSTRAINT ALL292 }293 elseif ($tokenizer->testUpperText('DISABLE') || $tokenizer->testUpperText('ENABLE'))294 {295 //(DISABLE|ENABLE) TRIGGER ALL296 }297 else298 {299 throw new NotSupportedException("'ADD' expected. line:".$tokenizer->getCurrentToken()->line);300 }301 }302 else303 {304 throw new NotSupportedException("'TABLE' expected. line:".$tokenizer->getCurrentToken()->line);305 }306 }307 /**308 * @param Tokenizer $tokenizer Statement tokens.309 *310 * @return void311 * @throws NotSupportedException312 */313 protected function executeCreateTable(Tokenizer $tokenizer)314 {315 $tokenizer->skipWhiteSpace();316 $this->tables->add(Table::create($tokenizer));317 }318 /**319 * @param Tokenizer $tokenizer Statement tokens.320 * @param boolean $unique Index uniqueness flag.321 *322 * @return void323 * @throws NotSupportedException324 */325 protected function executeCreateIndex(Tokenizer $tokenizer, $unique)326 {327 $tokenizer->skipWhiteSpace();328 $tokenizer->setBookmark();329 Index::searchTableName($tokenizer);330 $tableName = $tokenizer->getCurrentToken()->text;331 /** @var Table $table */332 $table = $this->tables->search($tableName);333 if (!$table)334 {335 throw new NotSupportedException("Table [$tableName] not found. line: ".$tokenizer->getCurrentToken()->line);336 }337 $tokenizer->restoreBookmark();338 $table->createIndex($tokenizer, $unique);339 }340 /**341 * @param Tokenizer $tokenizer Statement tokens.342 *343 * @return void344 * @throws NotSupportedException345 */346 protected function executeCreateTrigger(Tokenizer $tokenizer)347 {348 $tokenizer->skipWhiteSpace();349 $tokenizer->setBookmark();350 Trigger::searchTableName($tokenizer);351 $tableName = $tokenizer->getCurrentToken()->text;352 /** @var Table $table */353 $table = $this->tables->search($tableName);354 if (!$table)355 {356 throw new NotSupportedException("Table [$tableName] not found. line: ".$tokenizer->getCurrentToken()->line);357 }358 $tokenizer->restoreBookmark();359 $table->createTrigger($tokenizer);360 }361 /**362 * @param Tokenizer $tokenizer Statement tokens.363 *364 * @return void365 * @throws NotSupportedException366 */367 protected function executeCreateProcedure(Tokenizer $tokenizer)368 {369 $tokenizer->putBack();370 $this->procedures->add(Procedure::create($tokenizer));371 }372 /**373 * @param Tokenizer $tokenizer Statement tokens.374 *375 * @return void376 * @throws NotSupportedException377 */378 protected function executeCreateSequence(Tokenizer $tokenizer)379 {380 $tokenizer->skipWhiteSpace();381 $this->sequences->add(Sequence::create($tokenizer));382 }383}...

Full Screen

Full Screen

TokenizerTest.php

Source:TokenizerTest.php Github

copy

Full Screen

...16 * @dataProvider getTokens17 */18 public function testScan($text, $delimiters, $expected)19 {20 $tokenizer = new Mustache_Tokenizer;21 $this->assertSame($expected, $tokenizer->scan($text, $delimiters));22 }23 public function getTokens()24 {25 return array(26 array(27 'text',28 null,29 array(30 array(31 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,32 Mustache_Tokenizer::LINE => 0,33 Mustache_Tokenizer::VALUE => 'text',34 ),35 ),...

Full Screen

Full Screen

tokenizer

Using AI Code Generation

copy

Full Screen

1require_once 'vendor/autoload.php';2use \mageekguy\atoum\tests\units\test;3use \mageekguy\atoum\asserters;4use \mageekguy\atoum\exceptions;5use \mageekguy\atoum\tools\variable;6use \mageekguy\atoum\tools\diffs;7use \mageekguy\atoum\tools\asserters\phpString;8use \mageekguy\atoum\tools\asserters\phpArray;9use \mageekguy\atoum\tools\asserters\phpObject;10use \mageekguy\atoum\tools\asserters\phpFloat;11use \mageekguy\atoum\tools\asserters\phpInteger;12use \mageekguy\atoum\tools\asserters\phpBoolean;13use \mageekguy\atoum\tools\asserters\phpResource;14use \mageekguy\atoum\tools\asserters\phpGenerator;15use \mageekguy\atoum\tools\asserters\phpClass;16use \mageekguy\atoum\tools\asserters\phpFunction;17use \mageekguy\atoum\tools\asserters\phpConstant;18use \mageekguy\atoum\tools\asserters\phpCall;19use \mageekguy\atoum\tools\asserters\phpException;20use \mageekguy\atoum\tools\asserters\phpVariable;21use \mageekguy\atoum\tools\asserters\phpSizeOf;22use \mageekguy\atoum\tools\asserters\phpOutput;23use \mageekguy\atoum\tools\asserters\phpError;24use \mageekguy\atoum\tools\asserters\phpFile;25use \mageekguy\atoum\tools\asserters\phpString\utf8;26use \mageekguy\atoum\tools\asserters\phpString\binary;27use \mageekguy\atoum\tools\asserters\phpString\castToString;

Full Screen

Full Screen

tokenizer

Using AI Code Generation

copy

Full Screen

1require_once 'vendor/autoload.php';2use atoum\atoum\report\fields\runner\tests\void\cli;3use atoum\atoum\report\fields\runner\tests\void\html;4use atoum\atoum\report\fields\runner\tests\void\text;5use atoum\atoum\report\fields\runner\tests\void\xunit;6use atoum\atoum\report\fields\runner\tests\void\yaml;7use atoum\atoum\report\fields\runner\tests\void\json;8use atoum\atoum\report\fields\runner\tests\void\csv;9use atoum\atoum\report\fields\runner\tests\void\cobertura;10use atoum\atoum\report\fields\runner\tests\void\sonarqube;11use atoum\atoum\report\fields\runner\tests\void\junit;12use atoum\atoum\report\fields\runner\failures\cli;13use atoum\atoum\report\fields\runner\failures\html;14use atoum\atoum\report\fields\runner\failures\text;15use atoum\atoum\report\fields\runner\failures\xunit;16use atoum\atoum\report\fields\runner\failures\yaml;17use atoum\atoum\report\fields\runner\failures\json;18use atoum\atoum\report\fields\runner\failures\csv;19use atoum\atoum\report\fields\runner\failures\cobertura;20use atoum\atoum\report\fields\runner\failures\sonarqube;21use atoum\atoum\report\fields\runner\failures\junit;22use atoum\atoum\report\fields\runner\outputs\cli;23use atoum\atoum\report\fields\runner\outputs\html;24use atoum\atoum\report\fields\runner\outputs\text;

Full Screen

Full Screen

tokenizer

Using AI Code Generation

copy

Full Screen

1$tokenizer = new Tokenizer();2$tokens = $tokenizer->tokenize(file_get_contents('1.php'));3var_dump($tokens);4$tokens = token_get_all(file_get_contents('2.php'));5var_dump($tokens);6$tokenizer = new Tokenizer();7$tokens = $tokenizer->tokenize(file_get_contents('3.php'));8var_dump($tokens);9$tokenizer = new Tokenizer();10$tokens = $tokenizer->tokenize(file_get_contents('4.php'));11var_dump($tokens);12$tokenizer = new Tokenizer();13$tokens = $tokenizer->tokenize(file_get_contents('5.php'));14var_dump($tokens);15$tokenizer = new Tokenizer();16$tokens = $tokenizer->tokenize(file_get_contents('6.php'));17var_dump($tokens);18$tokenizer = new Tokenizer();19$tokens = $tokenizer->tokenize(file_get_contents('7.php'));20var_dump($tokens);21$tokenizer = new Tokenizer();22$tokens = $tokenizer->tokenize(file_get_contents('8.php'));23var_dump($tokens);24$tokenizer = new Tokenizer();25$tokens = $tokenizer->tokenize(file_get_contents('9.php'));26var_dump($tokens);27$tokenizer = new Tokenizer();28$tokens = $tokenizer->tokenize(file_get_contents('10.php'));29var_dump($tokens);30$tokenizer = new Tokenizer();31$tokens = $tokenizer->tokenize(file_get_contents('11.php'));32var_dump($tokens);33$tokenizer = new Tokenizer();

Full Screen

Full Screen

tokenizer

Using AI Code Generation

copy

Full Screen

1require_once 'vendor/autoload.php';2use \mageekguy\atoum\scripts\tokenizer;3$tokenizer = new tokenizer();4$tokenizer->addDirectory(__DIR__.'/src');5$tokenizer->run();6require_once 'vendor/autoload.php';7use \mageekguy\atoum\scripts\coverage;8$coverage = new coverage();9$coverage->addDirectory(__DIR__.'/src');10$coverage->run();11require_once 'vendor/autoload.php';12use \mageekguy\atoum\scripts\test;13$test = new test();14$test->addDirectory(__DIR__.'/src');15$test->run();16require_once 'vendor/autoload.php';17use \mageekguy\atoum\scripts\coverage;18$coverage = new coverage();19$coverage->addDirectory(__DIR__.'/src');20$coverage->run();21require_once 'vendor/autoload.php';22use \mageekguy\atoum\scripts\coverage;23$coverage = new coverage();24$coverage->addDirectory(__DIR__.'/src');25$coverage->run();26require_once 'vendor/autoload.php';27use \mageekguy\atoum\scripts\coverage;28$coverage = new coverage();29$coverage->addDirectory(__DIR__.'/src');30$coverage->run();31require_once 'vendor/autoload.php';32use \mageekguy\atoum\scripts\coverage;33$coverage = new coverage();34$coverage->addDirectory(__DIR__.'/src');35$coverage->run();36require_once 'vendor/autoload.php';37use \mageekguy\atoum\scripts\coverage;38$coverage = new coverage();39$coverage->addDirectory(__DIR__.'/src');

Full Screen

Full Screen

tokenizer

Using AI Code Generation

copy

Full Screen

1$tokens = new atoum\atoum\cli\tokens();2$tokens->addToken('1', '1');3$tokens->addToken('2', '2');4$tokens->addToken('3', '3');5$tokens->addToken('4', '4');6$tokens->addToken('5', '5');7$tokens->addToken('6', '6');8$tokens->addToken('7', '7');9$tokens->addToken('8', '8');10$tokens->addToken('9', '9');11$tokens->addToken('plus', '+');12$tokens->addToken('minus', '-');13$tokens->addToken('multiply', '*');14$tokens->addToken('divide', '/');15$tokens->addToken('leftParenthesis', '(');16$tokens->addToken('rightParenthesis', ')');17$tokens->addToken('number', '[0-9]+', function($value) {return (int) $value;});18$tokens->addToken('whitespace', '\s+');19$tokens->addToken('end', '$');20$lexer = new atoum\atoum\cli\lexer($tokens);21$tokens = new atoum\atoum\cli\tokens();22$tokens->addToken('1', '1');23$tokens->addToken('2', '2');24$tokens->addToken('3', '3');25$tokens->addToken('4', '4');26$tokens->addToken('5', '5');27$tokens->addToken('6', '6');28$tokens->addToken('7', '7');29$tokens->addToken('8', '8');30$tokens->addToken('9', '9');31$tokens->addToken('plus', '+');32$tokens->addToken('minus', '-');33$tokens->addToken('multiply', '*');34$tokens->addToken('divide', '/');35$tokens->addToken('leftParenthesis', '(');36$tokens->addToken('rightParenthesis', ')');37$tokens->addToken('number', '[0-9]+', function($value) {return (int) $value;});38$tokens->addToken('whitespace', '\s+');39$tokens->addToken('end', '$');

Full Screen

Full Screen

tokenizer

Using AI Code Generation

copy

Full Screen

1require_once 'vendor/autoload.php';2use \mageekguy\atoum\asserters;3$php = new \mageekguy\atoum\php\tokenizer();4$php->parseFile('2.php');5$parser = (new PhpParser\ParserFactory)->create(PhpParser\ParserFactory::PREFER_PHP7);6try {7 $stmts = $parser->parse(file_get_contents('2.php'));8} catch (PhpParser\Error $e) {9 echo 'Parse Error: ', $e->getMessage();10}11$traverser = new PhpParser\NodeTraverser;12$traverser->addVisitor(new PhpParser\NodeVisitor\NameResolver);13$traverser->addVisitor(new MyNodeVisitor);14$ast = $traverser->traverse($stmts);15$prettyPrinter = new PhpParser\PrettyPrinter\Standard;16echo $prettyPrinter->prettyPrintFile($ast);17$traverser = new PhpParser\NodeTraverser;18$traverser->addVisitor(new PhpParser\NodeVisitor\NameResolver);19$traverser->addVisitor(new MyNodeVisitor2);20$ast = $traverser->traverse($stmts);21$prettyPrinter = new PhpParser\PrettyPrinter\Standard;22echo $prettyPrinter->prettyPrintFile($ast);23$traverser = new PhpParser\NodeTraverser;24$traverser->addVisitor(new PhpParser\NodeVisitor\NameResolver);25$traverser->addVisitor(new MyNodeVisitor3);26$ast = $traverser->traverse($stmts);27$prettyPrinter = new PhpParser\PrettyPrinter\Standard;28echo $prettyPrinter->prettyPrintFile($ast);29$traverser = new PhpParser\NodeTraverser;30$traverser->addVisitor(new PhpParser\NodeVisitor\NameResolver);31$traverser->addVisitor(new MyNodeVisitor4);32$ast = $traverser->traverse($stmts);33$prettyPrinter = new PhpParser\PrettyPrinter\Standard;

Full Screen

Full Screen

tokenizer

Using AI Code Generation

copy

Full Screen

1$tokenizer = new \mageekguy\atoum\tokenizer();2$tokens = $tokenizer->parseFile('2.php');3$tokens = $tokenizer->parseString('<?php echo "Hello World"; ?>');4$tokenizer = new \mageekguy\atoum\tokenizer();5$tokens = $tokenizer->parseFile('2.php');6$tokens = $tokenizer->parseString('<?php echo "Hello World"; ?>');7$tokenizer = new \mageekguy\atoum\tokenizer();8$tokens = $tokenizer->parseFile('3.php');9$tokens = $tokenizer->parseString('<?php echo "Hello World"; ?>');10$tokenizer = new \mageekguy\atoum\tokenizer();11$tokens = $tokenizer->parseFile('4.php');12$tokens = $tokenizer->parseString('<?php echo "Hello World"; ?>');13$tokenizer = new \mageekguy\atoum\tokenizer();14$tokens = $tokenizer->parseFile('5.php');15$tokens = $tokenizer->parseString('<?php echo "Hello World"; ?>');16$tokenizer = new \mageekguy\atoum\tokenizer();17$tokens = $tokenizer->parseFile('6.php');18$tokens = $tokenizer->parseString('<?php echo "Hello World"; ?>');19$tokenizer = new \mageekguy\atoum\tokenizer();20$tokens = $tokenizer->parseFile('7.php');21$tokens = $tokenizer->parseString('<?php echo "Hello World

Full Screen

Full Screen

tokenizer

Using AI Code Generation

copy

Full Screen

1$tokenizer = new atoum\php\tokenizer(file_get_contents('1.php'));2$tokenizer->parse();3$tokens = $tokenizer->getTokens();4$tokens->rewind();5while ($tokens->valid()) {6 $token = $tokens->current();7 if ($token[0] == T_STRING) {8";9 }10 $tokens->next();11}12$tokenizer = new atoum\php\tokenizer(file_get_contents('2.php'));13$tokenizer->parse();14$tokens = $tokenizer->getTokens();15$tokens->rewind();16while ($tokens->valid()) {17 $token = $tokens->current();18 if ($token[0] == T_STRING) {19";20 }21 $tokens->next();22}23$tokenizer = new atoum\php\tokenizer(file_get_contents('3.php'));24$tokenizer->parse();25$tokens = $tokenizer->getTokens();26$tokens->rewind();27while ($tokens->valid()) {28 $token = $tokens->current();29 if ($token[0] == T_STRING) {30";31 }32 $tokens->next();33}34$tokenizer = new atoum\php\tokenizer(file_get_contents('4.php'));35$tokenizer->parse();36$tokens = $tokenizer->getTokens();37$tokens->rewind();38while ($tokens->valid()) {39 $token = $tokens->current();40 if ($token[0] == T_STRING) {41";42 }43 $tokens->next();44}45$tokenizer = new atoum\php\tokenizer(file_get_contents('5.php'));46$tokenizer->parse();

Full Screen

Full Screen

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.

LambdaTest Learning Hubs:

YouTube

You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.

Run Atoum automation tests on LambdaTest cloud grid

Perform automation testing on 3000+ real desktop and mobile devices online.

... View sample repo... View DocumentationSign up Free
›
_

Most used methods in tokenizer

Run Selenium Automation Tests on LambdaTest Cloud Grid

Trigger Selenium automation tests on a cloud-based Grid of 3000+ real browsers and operating systems.

Test now for Free

Try LambdaTest Now !!

Get 100 minutes of automation test minutes FREE!!

...
Sign up with GoogleSign up with Email
Next-Gen App & Browser Testing Cloud

Was this article helpful?

Helpful

NotHelpful