<?php


/**
* Tokenizer 
*
* @abstract 
* This class just tidies up the tokens (adds a few things together.. to try and help!)
*
* @version    $Id: php_phpdoc_class,v 1.1 2002/04/22 10:20:29 alan_k Exp $
*/

	class Tokenizer
	{
		var $tokens = array();
		var $code;
		function Parse($file)
		{
			$this->code = file_get_contents($file);
			$this->Tokenize();
       
		}
		function Tokenize()
		{
			$tokens = token_get_all($this->code);
			foreach($tokens as $token)
			{
				if(is_array($token))
					$this->tokens[] = $token;
				else
					$this->tokens[] = array($token, null);
			}
		}
		function Strip()
		{
			$strip = array(T_WHITESPACE, T_COMMENT);
			$tokens = array();
            for ($i=0;$i<count($this->tokens);$i++) {
                $token = $this->tokens[$i];
                // if(is_array($token)){
				//	echo token_name((int) $token[0]), ' -- \'', $token[1], '\'', "\n";
                //}
                switch (true) {
                    // skip comments and whitespace.
                    case (in_array($this->tokens[$i][0], $strip)):
                        break;
                    
                    case (
                        ($this->tokens[$i][0]   == T_STRING) &&
                        ($this->tokens[$i+1][0] == T_DOUBLE_COLON) &&
                        ($this->tokens[$i+2][0] == T_STRING)):
                        $tokens[] = array(
                            T_STRING,
                            $this->tokens[$i][1].'::'.$this->tokens[$i+2][1]
                            );
                        $i +=2;
                        break;
                        
                    default:
                        $tokens[] = $this->tokens[$i];
                }
			}
			$this->tokens = $tokens;
            //$this->Display();
		}
		function Display()
		{
			foreach($this->tokens as $token)
			{
				if(is_array($token))
					echo token_name((int) $token[0]), ' -- \'', $token[1], '\'', "\n";
			}
		}
		function Reassemble()
		{
			$code = (string)null;
			foreach($this->tokens as $token)
			{
				if(is_array($token))
					$code .= $token[1];
				else
					$code .= $token;
			}
			return $code;
		}
	}