View Javadoc

    1   /*
    2    * PythonTokenMarker.java - Python token marker
    3    * Copyright (C) 1999 Jonathan Revusky
    4    * Copyright (C) 1998, 1999 Slava Pestov
    5    *
    6    * You may use and modify this package for any purpose. Redistribution is
    7    * permitted, in both source and binary form, provided that this notice
    8    * remains intact in all source distributions of this package.
    9    */
    10
    11  package org.syntax.jedit.tokenmarker;
    12
    13  import javax.swing.text.Segment;
    14
    15  import org.syntax.jedit.KeywordMap;
    16  import org.syntax.jedit.SyntaxUtilities;
    17
    18  /***
    19   * Python token marker.
    20   *
    21   * @author Jonathan Revusky
    22   * @version $Id: PythonTokenMarker.java,v 1.3 1999/12/14 04:20:35 sp Exp $
    23   */
    24  public class PythonTokenMarker extends TokenMarker
    25  {
    26  	private static final byte TRIPLEQUOTE1 = Token.INTERNAL_FIRST;
    27  	private static final byte TRIPLEQUOTE2 = Token.INTERNAL_LAST;
    28
    29  	public PythonTokenMarker()
    30  	{
    31  		this.keywords = getKeywords();
    32  	}
    33
    34  	public byte markTokensImpl(byte token, Segment line, int lineIndex)
    35  	{
    36  		char[] array = line.array;
    37  		int offset = line.offset;
    38  		lastOffset = offset;
    39  		lastKeyword = offset;
    40  		int length = line.count + offset;
    41  		boolean backslash = false;
    42
    43  loop:		for(int i = offset; i < length; i++)
    44  		{
    45  			int i1 = (i+1);
    46
    47  			char c = array[i];
    48  			if(c == '//')
    49  			{
    50  				backslash = !backslash;
    51  				continue;
    52  			}
    53
    54  			switch(token)
    55  			{
    56  			case Token.NULL:
    57  				switch(c)
    58  				{
    59  				case '#':
    60  					if(backslash)
    61  						backslash = false;
    62  					else
    63  					{
    64  						doKeyword(line,i,c);
    65  						addToken(i - lastOffset,token);
    66  						addToken(length - i,Token.COMMENT1);
    67  						lastOffset = lastKeyword = length;
    68  						break loop;
    69  					}
    70  					break;
    71  				case '"':
    72  					doKeyword(line,i,c);
    73  					if(backslash)
    74  						backslash = false;
    75  					else
    76  					{
    77  						addToken(i - lastOffset,token);
    78  						if(SyntaxUtilities.regionMatches(false,
    79  							line,i1,"\"\""))
    80  						{
    81  							token = TRIPLEQUOTE1;
    82  						}
    83  						else
    84  						{
    85  							token = Token.LITERAL1;
    86  						}
    87  						lastOffset = lastKeyword = i;
    88  					}
    89  					break;
    90  				case '\'':
    91  					doKeyword(line,i,c);
    92  					if(backslash)
    93  						backslash = false;
    94  					else
    95  					{
    96  						addToken(i - lastOffset,token);
    97  						if(SyntaxUtilities.regionMatches(false,
    98  							line,i1,"''"))
    99  						{
    100 							token = TRIPLEQUOTE2;
    101 						}
    102 						else
    103 						{
    104 							token = Token.LITERAL2;
    105 						}
    106 						lastOffset = lastKeyword = i;
    107 					}
    108 					break;
    109 				default:
    110 					backslash = false;
    111 					if(!Character.isLetterOrDigit(c)
    112 						&& c != '_')
    113 						doKeyword(line,i,c);
    114 					break;
    115 				}
    116 				break;
    117 			case Token.LITERAL1:
    118 				if(backslash)
    119 					backslash = false;
    120 				else if(c == '"')
    121 				{
    122 					addToken(i1 - lastOffset,token);
    123 					token = Token.NULL;
    124 					lastOffset = lastKeyword = i1;
    125 				}
    126 				break;
    127 			case Token.LITERAL2:
    128 				if(backslash)
    129 					backslash = false;
    130 				else if(c == '\'')
    131 				{
    132 					addToken(i1 - lastOffset,Token.LITERAL1);
    133 					token = Token.NULL;
    134 					lastOffset = lastKeyword = i1;
    135 				}
    136 				break;
    137 			case TRIPLEQUOTE1:
    138 				if(backslash)
    139 					backslash = false;
    140 				else if(SyntaxUtilities.regionMatches(false,
    141 					line,i,"\"\"\""))
    142 				{
    143 					addToken((i+=4) - lastOffset,
    144 						Token.LITERAL1);
    145 					token = Token.NULL;
    146 					lastOffset = lastKeyword = i;
    147 				}
    148 				break;
    149 			case TRIPLEQUOTE2:
    150 				if(backslash)
    151 					backslash = false;
    152 				else if(SyntaxUtilities.regionMatches(false,
    153 					line,i,"'''"))
    154 				{
    155 					addToken((i+=4) - lastOffset,
    156 						Token.LITERAL1);
    157 					token = Token.NULL;
    158 					lastOffset = lastKeyword = i;
    159 				}
    160 				break;
    161 			default:
    162 				throw new InternalError("Invalid state: "
    163 					+ token);
    164 			}
    165 		}
    166
    167 		switch(token)
    168 		{
    169 			case TRIPLEQUOTE1:
    170 			case TRIPLEQUOTE2:
    171 				addToken(length - lastOffset,Token.LITERAL1);
    172 				break;
    173 			case Token.NULL:
    174 				doKeyword(line,length,'\0');
    175 			default:
    176 				addToken(length - lastOffset,token);
    177 				break;
    178 		}
    179
    180 		return token;
    181 	}
    182
    183 	public static KeywordMap getKeywords()
    184 	{
    185 		if (pyKeywords == null)
    186 		{
    187 			pyKeywords = new KeywordMap(false);
    188 			pyKeywords.add("and",Token.KEYWORD3);
    189 			pyKeywords.add("not",Token.KEYWORD3);
    190 			pyKeywords.add("or",Token.KEYWORD3);
    191 			pyKeywords.add("if",Token.KEYWORD1);
    192 			pyKeywords.add("for",Token.KEYWORD1);
    193 			pyKeywords.add("assert",Token.KEYWORD1);
    194 			pyKeywords.add("break",Token.KEYWORD1);
    195 			pyKeywords.add("continue",Token.KEYWORD1);
    196 			pyKeywords.add("elif",Token.KEYWORD1);
    197 			pyKeywords.add("else",Token.KEYWORD1);
    198 			pyKeywords.add("except",Token.KEYWORD1);
    199 			pyKeywords.add("exec",Token.KEYWORD1);
    200 			pyKeywords.add("finally",Token.KEYWORD1);
    201 			pyKeywords.add("raise",Token.KEYWORD1);
    202 			pyKeywords.add("return",Token.KEYWORD1);
    203 			pyKeywords.add("try",Token.KEYWORD1);
    204 			pyKeywords.add("while",Token.KEYWORD1);
    205 			pyKeywords.add("def",Token.KEYWORD2);
    206 			pyKeywords.add("class",Token.KEYWORD2);
    207 			pyKeywords.add("del",Token.KEYWORD2);
    208 			pyKeywords.add("from",Token.KEYWORD2);
    209 			pyKeywords.add("global",Token.KEYWORD2);
    210 			pyKeywords.add("import",Token.KEYWORD2);
    211 			pyKeywords.add("in",Token.KEYWORD2);
    212 			pyKeywords.add("is",Token.KEYWORD2);
    213 			pyKeywords.add("lambda",Token.KEYWORD2);
    214 			pyKeywords.add("pass",Token.KEYWORD2);
    215 			pyKeywords.add("print",Token.KEYWORD2);
    216 		}
    217 		return pyKeywords;
    218 	}
    219
    220 	// private members
    221 	private static KeywordMap pyKeywords;
    222
    223 	private KeywordMap keywords;
    224 	private int lastOffset;
    225 	private int lastKeyword;
    226
    227 	private boolean doKeyword(Segment line, int i, char c)
    228 	{
    229 		int i1 = i+1;
    230
    231 		int len = i - lastKeyword;
    232 		byte id = keywords.lookup(line,lastKeyword,len);
    233 		if(id != Token.NULL)
    234 		{
    235 			if(lastKeyword != lastOffset)
    236 				addToken(lastKeyword - lastOffset,Token.NULL);
    237 			addToken(len,id);
    238 			lastOffset = i;
    239 		}
    240 		lastKeyword = i1;
    241 		return false;
    242 	}
    243 }