Syntax Highlighting classes with example

BlitzMax Forums/BlitzMax Programming/Syntax Highlighting classes with example

ozak	(Posted 2006) [#1]

Yup. After some quick hacking I got an example running very well. Nice clean written classes and example. No optimizations done yet mind you (except where obvious).

I'm thinking about adding it to the sample code section later after you guys have checked it out. (And myself)

A lot of people talked about the speed of the syntax highlighter, and I haven't really tested it with large files.
However, I only update the current line on each keystroke which seems plenty fast on my systems :)

First the example code. Show's how easy it is to use.
Save it as whatever you want. It's your main file.

' LUA Subset Syntax Highlighter usage example

' Include needed stuff
Include "LineTokenizer.bmx"
Include "SyntaxHighLighter.bmx"

' Window setup from MaxGUI tutorials
Local MyWindow:TGadget=CreateWindow("Simple LUA Subset Syntax Highlighting Example", 40,40,400,400,Null,0|WINDOW_TITLEBAR|WINDOW_RESIZABLE)
Global MyText:TGadget=CreateTextArea(0,0,GadgetWidth(MyWindow),GadgetHeight(MyWindow),MyWindow)	
SetGadgetLayout(MyText, 1,1,1,1)

' Create tokenizer
myTokenizer:TLineTokenizer= New TLineTokenizer
myTokenizer.Init(",= ().")

' Setup highlighter
myHighlighter:TSyntaxHighLighter = New TSyntaxHighLighter
myHighLighter.Init(MyText, myTokenizer)
myHighLighter.SetFont("Courier New",12)
myHighLighter.AddKeyword("print")
myHighLighter.AddKeyword("do")
myHighLighter.AddKeyword("for")
myHighLighter.AddKeyword("end")
myHighLighter.AddKeyword("math")
myHighLighter.AddKeyword("random")

' Load from script
myHighLighter.LoadAndColorFromFile("script.txt")

' Current cursor pos
Global cursorPos:Int = 0

' Main loop
Repeat

  WaitEvent()

  Select EventID()

	  Case EVENT_WINDOWCLOSE
	     End
	
	  Case EVENT_GADGETSELECT
	     cursorPos=TextAreaCursor(MyText)

  End Select

  ' Lock for a minor speed boost.
  LockTextArea(MyText)
  myHighlighter.ColorLine(TextAreaLine(MyText, cursorpos))
  UnlockTextArea(MyText)
  
Forever
End

script.txt

for i = 1, 10 do
	print(math.random(6))
end

SyntaxHighlighter.bmx

' Syntax highligher class by Odin Jensen (http://www.furi.dk)
Type TSyntaxHighLighter

	' Internal tokenizer
	Field tokenizer:TLineTokenizer = Null
	
	' Gadget
	Field control:TGadget = Null
	
	' Colors
	Field textR:Byte = 255
	Field textG:Byte = 255
	Field textB:Byte = 50
	
	Field sepR:Byte = 0
	Field sepG:Byte = 255
	Field sepB:Byte = 0
	
	Field numberR:Byte = 0
	Field numberG:Byte = 255
	Field numberB:Byte = 255
	
	Field keywordR:Byte = 255
	Field keywordG:Byte = 255
	Field keywordB:Byte = 255
	
	' Keyword list
	Field keywords:TList = New TList

	
	' Init 
	Method Init(control:TGadget, tokenizer:TLineTokenizer)
	
		' Store for later
		Self.control = control
		Self.tokenizer = tokenizer		
		
		' Set default (or userdefined) colors
		SetBackgroundColor(1,81,107)
		SetTextColor(textR, textG, textB)
		SetSeperatorColor(sepR, sepG, sepB)
		SetNumberColor(numberR, numberG, numberB)
		SetKeywordColor(keywordR, keywordG, keywordB)	
		
		' Set tab size
		SetTextAreaTabs(control, 2)
	
	End Method
	
	' Set background color
	Method SetBackgroundColor(r:Byte, g:Byte, b:Byte)
		
		SetTextAreaColor(control,r,g,b,True)

	End Method
	
	' Set text color
	Method SetTextColor(r:Byte, g:Byte, b:Byte)
	
		SetTextAreaColor(control,r,g,b,False)
		
		textR = r
		textG = g
		textB = b
	
	End Method
	
	' Set seperator
	Method SetSeperatorColor(r:Byte, g:Byte, b:Byte)
	
		sepR = r
		sepG = g
		sepB = b
		
	End Method
	
	' Set number color
	Method SetNumberColor(r:Byte, g:Byte, b:Byte)
	
		numberR = r
		numberG = g
		numberB = b
		
	End Method
	
	' Set keyword color
	Method SetKeywordColor(r:Byte, g:Byte, b:Byte)
	
		keywordR = r
		keywordG = g
		keywordB = b
		
	End Method
	
	' Set font
	Method SetFont(name:String, size:Int)
	
		Local GUIFont:TGuiFont=LoadGuiFont(name, size)
		SetTextAreaFont(control,GUIFont)

	
	End Method
	
	' Add keyword
	Method AddKeyWord(word:String)
	
		keywords.addLast(word)
	
	End Method
	
	' Highlight from file
	Method LoadAndColorFromFile:Int(file:String)
	
		' Read script and add lines
		Local in:TStream=ReadStream(file)
		If (in = Null)
			Return False
		End If
		
		' Lock TextArea
		LockTextArea(control)
		
		' Do file
		Local curLine:Int = 0
		While Not Eof(in)
		    Local line:String = ReadLine(in)
		
			' Add line to control
			AddTextAreaText(control,line+"~n")	

			' Color line
			ColorLine(curLine)			
			
			' Advance line
			curLine:+1
		Wend
		CloseStream(in)
		
		' Unlock TextArea
		UnlockTextArea(control)
		
		' All went well
		Return True
		
	End Method		
	
	' Color line
	Method ColorLine(lineNum:Int)
		
		' Find line pos
		Local curPos:Int = TextAreaChar(control, lineNum)
		
		' Get current line
		Local line:String = TextAreaText(control, lineNum, 1, TEXTAREA_LINES).Replace("~n", "")
			
		' Reset tokenizer
		tokenizer.Reset()
		
		' Tokenize line
		tokenizer.Tokenize(line)
		Local seps:TList = tokenizer.GetSeperators()
	
		' Do seperators
		For Local cs:String = EachIn seps

			For Local i:Int = 0 To line.length-1

				If (Chr(line[i]) = cs)
			
					FormatTextAreaText(control, sepR, sepG, sepB, 0,curPos+i,1)
				
				End If
			Next

		Next	
		
		' Do tokens
		Local tokens:TList = tokenizer.GetTokens()	
		For Local ct:String = EachIn tokens

			If (line.find(ct) <> -1)
		
				If (IsNumber(ct))
					FormatTextAreaText(control, numberR, numberG, numberB, 0,curPos+line.find(ct),ct.length)
				Else If (IsKeyword(ct))
					FormatTextAreaText(control, keywordR, keywordG, keywordB, 0,curPos+line.find(ct),ct.length)
				Else
					FormatTextAreaText(control, textR, textG, textB, 0,curPos+line.find(ct),ct.length)
				End If

			End If
				
		Next	

	
	End Method
	
	' Private IsNumber
	Method IsNumber:Int(in:String)
	
		Return (in = "0" Or in.toInt() <> 0)
	
	End Method
	
	' Private IsKeyword
	Method IsKeyword:Int(in:String)
			
		For Local ck:String = EachIn keywords
					
			If (ck.toLower() = in.toLower())
			
				Return True
			
			End If
		
		Next
		
		Return False
	
	End Method

End Type

LineTokenizer.bmx

' Simple line based tokenizer by Odin Jensen (http://www.furi.dk)
Type TLineTokenizer

	' Token list
	Field tokens:TList = New TList
	
	' Seperator list
	Field seps:TList = New TList
	
	' Seperator search list
	Field sepSearchList:String = " "

	' Init tokenizer
	Method Init(sepList:String)
	
		Reset()
		sepSearchList = sepList
	
	End Method
	
	' Reset tokenizer
	Method Reset()
	
		tokens = New TList
		seps = New TList

	End Method
	
	' Tokenize
	Method Tokenize(line:String)
	
		Local lastSepPos:Int = 0
		For Local i:Int = 0 To line.length-1
		
			If (IsSep(Chr(line[i])))
			
				Local temp:String = line[lastSepPos..i]
				temp = StripIlligalChars(temp)
				If (temp.length > 0)
					tokens.addLast(temp)
				End If
				
				If (seps.Contains(Chr(line[i])) = False And Chr(line[i]) <> " ")				
					seps.addLast(Chr(line[i]))
				End If
				lastSepPos = i+1
			
			End If
					
		Next
		
		' Any leftovers?
		If (lastSepPos < line.length)
		
			tokens.addLast(line[lastSepPos..line.length])
		
		End If
	
	End Method	
	
	' Get tokens
	Method GetTokens:TList()
		
		Return tokens
	
	End Method
	
	' Get non-space seperators
	Method GetSeperators:TList()
	
		Return seps
		
	End Method		
		
	' Private check for seperator function
	Method IsSep:Int(ch:String)
	
		If (sepSearchList.find(ch) <> -1)
		
			Return True		
				
		End If
		
		Return False
	
	End Method
	
	' Private strip illigal chars function
	Method StripIlligalChars:String(in:String)

		Local ret:String = in.replace("~t", "")
		ret = ret.replace("~n", "")
		Return ret	
			
	End Method


End Type

Rem 
' Test
myTokenizer:TLineTokenizer = New TLineTokenizer

myTokenizer.Init(",= ()")
myTokenizer.Tokenize("for i = 0, 10 do")
myTokenizer.Tokenize("~tprint(Dice(6)")
myTokenizer.Tokenize("end")

tokens:TList = myTokenizer.GetTokens()
seps:Tlist = myTokenizer.GetSeperators()

For Local ct:String = EachIn tokens

	Print("Token: " + ct)

Next

Print("~n")

For Local cs:String = EachIn seps

	Print("Seps: " + cs)

Next

End Rem

Mark Tiffany

(Posted 2006) [#2]

I haven't checked this out, but I'm interested in this from the POV of the Community IDE. It looks like it would be failry easy to have different syntax highlighters for different types of code (e.g. in the IDE have BMX, C++, HTML , etc code highlighted differently in each tab).

The only thing is that the highlighter at the moment does a lot more than just tokenise. E.g. Highlighting comments, string literals, numeric literals differently. It also does (or should do in the same pass) identify types, functions, etc for populatingt he code structure lists. (and equally could be used for self-highlighting). Another thing I just spotted is that the only separator looks to be " ", you'll need a few more...

What do you reckon to taking this up a level and seeing if we can use it in the Community IDE? I'll take a look at the code and performance over the weekend to see how it might compare...

taxlerendiosk

(Posted 2006) [#3]

Thank you, I've been looking forward to trying this...

Do you know why you don't seem to be able to select with the mouse?

ozak	(Posted 2006) [#4]

Hi Mark

You can add any seperator you like as per the example.

' Create tokenizer
myTokenizer:TLineTokenizer= New TLineTokenizer
myTokenizer.Init(",= ().")

' Setup highlighter
myHighlighter:TSyntaxHighLighter = New TSyntaxHighLighter
myHighLighter.Init(MyText, myTokenizer)
myHighLighter.SetFont("Courier New",12)
myHighLighter.AddKeyword("print")
myHighLighter.AddKeyword("do")
myHighLighter.AddKeyword("for")
myHighLighter.AddKeyword("end")
myHighLighter.AddKeyword("math")
myHighLighter.AddKeyword("random")

There are huge speedups to be made, but I didn't look to hard at the color coding speed, since my major idea was to only color the line you're currently writing.
Loading the code file just uses the same ColorLine code.

Actually I've been thinking about the community IDE as I'm working on Code Folding too. I got an idea where you can code fold any block of text you like and it just shows a + with the topmost line of code and some ... at the end.

Maybe, but I'm a busy man :)

As to why the mouse selection does not work (you can however, doubleclick on a word and have it selected) it is definately my syntax highlighters fault.
I'll look into this when I get some time for it ;)

klepto2

(Posted 2006) [#5]

Nice one, I have tried various things by myself. And had a good working highlighter until i've tried to speed it up (I will have a look at my code later on). It was relativly fast.
was highlighting a code of ~120 kb in ~5 seconds (with ~13000 Keywords).
A little speedup to your highlighter would be to use TMap instead of TList for your Keywords. And a suggestion to enhance your highlighter is to make separate Keyword type, where you store the Color and Style. So you will be able to add different Colors to Keywords.

Here is a TMapped Version of 'syntaxhighlighter.bmx'


' Syntax highligher class by Odin Jensen (http://www.furi.dk)
Type TSyntaxHighLighter

	' Internal tokenizer
	Field tokenizer:TLineTokenizer = Null
	
	' Gadget
	Field control:TGadget = Null
	
	' Colors
	Field textR:Byte = 255
	Field textG:Byte = 255
	Field textB:Byte = 255
	
	Field sepR:Byte = 0
	Field sepG:Byte = 255
	Field sepB:Byte = 0
	
	Field numberR:Byte = 0
	Field numberG:Byte = 255
	Field numberB:Byte = 255
	
	Field keywordR:Byte = 255
	Field keywordG:Byte = 255
	Field keywordB:Byte = 255
	
	' Keyword list
	Field keywords:TMap = New TMap

	
	' Init 
	Method Init(control:TGadget, tokenizer:TLineTokenizer)
	
		' Store for later
		Self.control = control
		Self.tokenizer = tokenizer		
		
		' Set default (or userdefined) colors
		SetBackgroundColor(1,81,107)
		SetTextColor(textR, textG, textB)
		SetSeperatorColor(sepR, sepG, sepB)
		SetNumberColor(numberR, numberG, numberB)
		SetKeywordColor(keywordR, keywordG, keywordB)	
		
		' Set tab size
		SetTextAreaTabs(control, 2)
	
	End Method
	
	' Set background color
	Method SetBackgroundColor(r:Byte, g:Byte, b:Byte)
		
		SetTextAreaColor(control,r,g,b,True)

	End Method
	
	' Set text color
	Method SetTextColor(r:Byte, g:Byte, b:Byte)
	
		SetTextAreaColor(control,r,g,b,False)
		
		textR = r
		textG = g
		textB = b
	
	End Method
	
	' Set seperator
	Method SetSeperatorColor(r:Byte, g:Byte, b:Byte)
	
		sepR = r
		sepG = g
		sepB = b
		
	End Method
	
	' Set number color
	Method SetNumberColor(r:Byte, g:Byte, b:Byte)
	
		numberR = r
		numberG = g
		numberB = b
		
	End Method
	
	' Set keyword color
	Method SetKeywordColor(r:Byte, g:Byte, b:Byte)
	
		keywordR = r
		keywordG = g
		keywordB = b
		
	End Method
	
	' Set font
	Method SetFont(name:String, size:Int)
	
		Local GUIFont:TGuiFont=LoadGuiFont(name, size)
		SetTextAreaFont(control,GUIFont)

	
	End Method
	
	' Add keyword
	Method AddKeyWord(word:String,r:Int,g:Int,b:Int,style:Int)
		Local key:TKeyword = New TKeyWord
		Key.name = word
		Key.r = r
		Key.g = g
		Key.b = b
		Key.style = style
		keywords.Insert(word,Key)
	End Method
	
	' Highlight from file
	Method LoadAndColorFromFile:Int(file:String)
	
		' Read script and add lines
		Local in:TStream=ReadStream(file)
		If (in = Null)
			Return False
		End If
		
		' Lock TextArea
		LockTextArea(control)
		
		' Do file
		Local curLine:Int = 0
		While Not Eof(in)
		    Local line:String = ReadLine(in)
		
			' Add line to control
			AddTextAreaText(control,line+"~n")	

			' Color line
			ColorLine(curLine)			
			
			' Advance line
			curLine:+1
		Wend
		CloseStream(in)
		
		' Unlock TextArea
		UnlockTextArea(control)
		
		' All went well
		Return True
		
	End Method		
	
	' Color line
	Method ColorLine(lineNum:Int)
		
		' Find line pos
		Local curPos:Int = TextAreaChar(control, lineNum)
		
		' Get current line
		Local line:String = TextAreaText(control, lineNum, 1, TEXTAREA_LINES).Replace("~n", "")
			
		' Reset tokenizer
		tokenizer.Reset()
		
		' Tokenize line
		tokenizer.Tokenize(line)
		Local seps:TList = tokenizer.GetSeperators()
	
		' Do seperators
		For Local cs:String = EachIn seps

			For Local i:Int = 0 To line.length-1

				If (Chr(line[i]) = cs)
			
					FormatTextAreaText(control, sepR, sepG, sepB, 0,curPos+i,1)
				
				End If
			Next

		Next	
		
		' Do tokens
		Local tokens:TList = tokenizer.GetTokens()	
		For Local ct:String = EachIn tokens
			Local Key:TKeyWord = IsKeyword(ct)

			If (line.find(ct) <> -1)
		
				If (IsNumber(ct))
					FormatTextAreaText(control, numberR, numberG, numberB, 0,curPos+line.find(ct),ct.length)
				Else If (Key) <> Null 
					FormatTextAreaText(control, key.R, key.G, key.B, Key.Style,curPos+line.find(ct),ct.length)
				Else
					FormatTextAreaText(control, textR, textG, textB, 0,curPos+line.find(ct),ct.length)
				End If

			End If
				
		Next	

	
	End Method
	
	' Private IsNumber
	Method IsNumber:Int(in:String)
	
		Return (in = "0" Or in.toInt() <> 0)
	
	End Method
	
	' Private IsKeyword
	Method IsKeyword:TKeyword(in:String)
			Local Key:TKeyword = TKeyword(Keywords.ValueforKey(in))
			Return Key	
	End Method

End Type

Type TKeyword
	Field r:Int = 255
	Field g:Int = 255
	Field b:Int = 50
	Field style:Int = 1
	Field name:String
End Type

The speed increase will be noticable if you have to check alot of Keywords like all BMax Module Keywords.

ozak	(Posted 2006) [#6]

Cool!
That was actually one of the speed fixes I had in mind :)

CASO	(Posted 2007) [#7]

This is awesome.

I was thinking about starting an IDE project and this is a great syntax highlighting foundation.