$ARGV = shift @ARGV;
my %tokens;

my @files = glob("$ARGV/*.rnc");

open( TOKEN, ">tokens.txt" ) || die "can't write token file";

foreach( @files )
{
	print( "parsing $_\n" );
	open ( XSD, $_ ) || die "can't open token file: $!";
	while( <XSD> )
	{
		chomp($_);
		if( /element (\S*:)?(\S*)/ )
		{		
			$tokens{$2} = 1;
			print(".");
		}
		elsif( /attribute (\S*:)?(\S*)/ )
		{
			$tokens{$2} = 1;
			print(".");
		}
		elsif( /list\s*\{/ )
		{
			while( <XSD> )
			{
				chomp($_);
				last if( /^\s*\}/ );
				if( /"(\S*?)\"/ )
				{
					$tokens{$1} = 1;
					print(".");
				}				
			}
		}
	}
	close ( XSD );
	
	print("\n" );
}

foreach( sort(keys(%tokens)) )
{
	print TOKEN "$_\n";
}
close( TOKEN );
