From 4140c872f5d4ec144ed284037f59780d8501d90d Mon Sep 17 00:00:00 2001
From: Paul Robson <credentials.helper>
Date: Thu, 4 Apr 2024 08:30:46 +0100
Subject: [PATCH] Decimal tokenisation fix

---
 .../__pycache__/tokeniser.cpython-311.pyc     | Bin 12840 -> 12904 bytes
 .../editing/tokenise/helpers/tokio.asm        |  27 +++++++++++++++---
 .../tokenise/workers/tok_intconstant.asm      |   3 +-
 basic/test.bsc                                |  20 +------------
 4 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/basic/scripts/__pycache__/tokeniser.cpython-311.pyc b/basic/scripts/__pycache__/tokeniser.cpython-311.pyc
index a2d19959e4aa0eb38f6568887904459ae40d1991..bc09b84835bb3d5bd6e77e389883d1b500b00556 100644
GIT binary patch
delta 1455
zcma)6&1(}u6rb7cX7ja4x|_`=Y1&QFrjpuPi)gi?ZKbFvw3P~iml~YbepsuUfFYz%
zwLM8`@f{QeT@k9-gO>aOUPTaEC`AH--o%6S>_MDKOl)El2l93@@8kF0@4cB9<O`9W
zOOhSYl>|I~YF#=ba|nYZw!@+v;4Wlgvt5JtqQ{avkwqpCiM>8pYIDO|xsx~4vby^f
zRZn3Sh^-M6=Ld}gbeG{eJL7eEA(=8JujlJ7&yumcJ()@vspQSc)C4psC-Ds!Q=+a1
z_=)qO(>%ZFZ`<;>!J^V$ee_C+_3Stix{tD$;6MS13BpYa?7M=E<thz7UC4=QZo{yq
zLCpJ{eM4dvuW|^j(O+@RaanE!-?9<w0zCBwp;rw%Y*%oGYUlWr1E$n2t^A*sYAHFI
zBIQJd`hdm}JA6_jj@_z$tBRR(103`De(h#`&I%jm{>bN|+LykgWg8K!S}%6OX-$tv
zHL|Ul0*qr1d{brE*3OEKVui303_!OYhaUkqc0tCkN8JA%&&9ELN=3fKn&zAO7?L}U
ze{(_VOXTU+jD*l-Zqx!hS_Fpl`U1QRu3!TsgDM;kRt7C|C{qq~Hzvw7&nPw)ve>jk
zzttNhE}~R54cG1j{5|b1dyu7lvLbEo0-dM<v`X>TN_|D80!jx&8L65((*X%N8X{_W
z{MIYSuV~;TzVZ~_W78MWYVuDfySPlLFj9A$4#As{mG&$lH#Z1bf%uC+Mqll+WfDl~
zr634Vhae0zc2_iy^2`gQ0UkF79G;S`S^o?0pk=;;#`eI~rt=<**!B;bp6Ylpij%P+
zv`VkqTKNemc<;Un6j`9i84N4lp1Ki4FBNM>R+k)=m20r3w^oDjg2{M@!n?2n@v!#~
zpV9EYd}25RJK<Q)ANkB-1->`$fsU309)jhTR?8r*F{6SRRbit=z?WdVWlZ}h+7{9q
zo@TRuQ*7N5Tj#qzSj1a1xkPM62u)BAW0HwRDw6CK35N{_I~dP>ZOd~w38Quu@*Ni&
zhuP_2CXP^%ubY~kycXvT_UIZzaBkl-wP3-LFvgi6Or&$R&N~>VVQGI9&*fh4Pvf6N
C;44}H

delta 1371
zcma)6OK1~O6rDHuO_EOYn@|3~CXH#eABumZEl8n)X-gHM3rpyfnx@(^3D|~ILalV;
zC-ve!t<Wm%!jhc}aVsKN=t3eE#f=LW>PmFud->7CRJ<^k;lA_EJ?FhU^F?|hv70Qb
zqj0?cx<A>y!(K4c+Z3>xqj1M;QyfcEux1wEb(;x(m^(wyaA6t|IEhgzOh@D%{1#Re
zREeY<n=2?MW~E73vJ9XRcxCa!Z%Z2-=UQlig)z?MSWhDf*^0YKk`(bGTNcTWV3-%-
zDYr?d5%e+wCuuhD+E%q2#<&SaVA5!V#w++K#Ud?fmtoZwhHc)Y*G-`mt}iN*EMx=+
zeC7qNTEp@+<y{bUh&1*$j6^tx4AAUqhAyj8TMH<XbfXg0xTs;}!8#83DHu=_Fyh%N
zYCy}zi|XWiM6tX1Z^d;aWg&=t=yGBeLzpmQy`nXgX<ig*jQ>qzD$z7Bv+wS!I5o_p
zs#i)fS5DB#q*#iYpvN`v(d~m?-T+xq30LhbvcQbpfpicx2lR&+u;9*SMB!`Z;tF!o
z@^TvG^w7_69?2kabvK6&p*h?@x6WQMFi>NFU&dX}QyaMFR{4r4s$!*5khoEz?Z61;
zny?+eQxf7T0>V?a(+EIIB4nLDkG0_)-~%fK40{E*?>+%emln@d)ZAiKrB#xxL@xI=
zwrjE{eH8|Z!gvR)yG}9(Csm27PRMvTFuGO91at07#tiuyO_hx~3#_&C@W6ehw9GE5
z>bj{3b4m+dD_1+^74lv@!Q^}k9#&gxN$m~S%gGUYemLT@z>1fv!$1EAK8Rcp@dXvX
zP)9!D`$QuSE(bzQ1zjQ;m6M5^vE(&K2cjqf`+-*VMeHxd!^;*PJV6zjg<x>9^_^O?
zGP5+3?b%g}dus7JrBw<a7~NgwttpTj*=0g|OeoWzUkhGWIb)bWo+inKQ-XY#Uazbt
zg{gd7dx1tt7}xP&?-+9pk?JXuPUBQiPS3_}gcUORLCO(W?Rd-=5k!+bMUq6)LVmFG
RHbM*Vrpu3R<-c{!qd(Vg6RiLM

diff --git a/basic/sources/editing/tokenise/helpers/tokio.asm b/basic/sources/editing/tokenise/helpers/tokio.asm
index 394f78c1a..8c544cd79 100644
--- a/basic/sources/editing/tokenise/helpers/tokio.asm
+++ b/basic/sources/editing/tokenise/helpers/tokio.asm
@@ -83,23 +83,42 @@ _TEILoop:
 		bcc 	_TEIEndMainID
 		jsr 	TOKGetNext 					; get and consume
 		jsr 	TOKToUpper 					; capitalise it and add it
-		jsr 	_TEIAddChar
+		jsr 	TEIAddChar
 		bra 	_TEILoop
 _TEIEndMainID:
 		cmp 	#"$" 						; do we have a string ?
 		bne 	_TEINotString
-		jsr 	_TEIAddChar 				; add it
+		jsr 	TEIAddChar 					; add it
 		jsr 	TOKGetNext 					; consume $
 		jsr 	TOKGet 						; get next
 _TEINotString:
 		cmp 	#"("						; do we have an array ?
 		bne 	_TEINotArray
-		jsr 	_TEIAddChar 				; add (
+		jsr 	TEIAddChar 					; add (
 		jsr 	TOKGetNext 					; consume (
 _TEINotArray:		
 		rts
 
-_TEIAddChar:
+; ************************************************************************************************
+;
+;					Extract sequence of digits into the tokElement buffer
+;
+; ************************************************************************************************
+
+TOKExtractDigits:
+		stz 	tokElement
+_TEDLoop:
+		jsr 	TOKGet 						; get next, check end of token text
+		beq 	_TEDEndDigits  				; end of line
+		jsr 	TOKIsDigit 		 			; check digit
+		bcc 	_TEDEndDigits
+		jsr 	TOKGetNext 					; get and consume and add
+		jsr 	TEIAddChar
+		bra 	_TEDLoop
+_TEDEndDigits:
+		rts
+
+TEIAddChar:
 		inc 	tokElement 					; add it.
 		ldx 	tokElement
 		sta 	tokElement,x
diff --git a/basic/sources/editing/tokenise/workers/tok_intconstant.asm b/basic/sources/editing/tokenise/workers/tok_intconstant.asm
index 730a497e7..528ec08da 100644
--- a/basic/sources/editing/tokenise/workers/tok_intconstant.asm
+++ b/basic/sources/editing/tokenise/workers/tok_intconstant.asm
@@ -20,8 +20,7 @@
 ; ************************************************************************************************
 
 TokeniseIntConst:
-		jsr 	TOKExtractIdentifier 		; extract an identifier into the tokElement Buffer
-
+		jsr 	TOKExtractDigits 			; extract an identifier into the tokElement Buffer
 		lda 	#tokElement & $FF	 		; set up to convert it.
 		sta 	ControlParameters+4
 		lda 	#tokElement >> 8
diff --git a/basic/test.bsc b/basic/test.bsc
index 2e4e26d6b..4c4810b3e 100644
--- a/basic/test.bsc
+++ b/basic/test.bsc
@@ -1,19 +1 @@
-0 ' library line
-0 ' another library line
-rect 10,10 solid ink 2 to 100,100 to 110,120 to 130,440 to 140,500
-edit
-cls
-for i = 0 to 100
-for j = 0 to 100
-	doke $FF04,i:doke $FF06,j:poke $FF08,c
-	call send.message(5,39)
-	c = (c + 1) & $FF
-next
-next
-end
-
-proc send.message(g,f)
-	while peek($FF00):wend
-	poke $FF01,f:poke $FF00,g
-	while peek($FF00):wend
-endproc	
\ No newline at end of file
+10 print 1.2