Tokenizer hash table

.gitignore

@@ -0,0 +1,7 @@
+*.asm
+*.o
+*.seed
+/preForth/crc10.forth
+/preForth/preForth
+/preForth/seedForth
+/preForth/__temp__.fs

preForth/Makefile

@@ -9,14 +9,19 @@ HOSTFORTH=gforth
 # ------------------------------------------------------------------------
 
 .PHONY=all
-all: preForth seedForth seedForthDemo.seed seedForthInteractive.seed
+all: \
+preForth \
+seedForth \
+seedForthDemo.seed \
+seedForthBoot.seed \
+seedForthInteractive.seed
 
 .PHONY=test
 test: runseedforthdemo runseedforthinteractive
 
 .PHONY=runseedforthdemo
 runseedforthdemo: seedForth seedForthDemo.seed
-	cat seedForthDemo.seed | ./seedForth
+	./seedForth seedForthDemo.seed
 
 .PHONY=runseedfortinteractive
 runseedforthinteractive: seedForth seedForthInteractive.seed
@@ -25,52 +30,66 @@ runseedforthinteractive: seedForth seedForthInteractive.seed
 UNIXFLAVOUR=$(shell uname -s)
 EXT=asm
 
-seedForth-i386.asm: seedForth-i386.pre preForth
-	cat seedForth-i386.pre | ./preForth >seedForth-i386.asm
-
-# preForth connected to stdin - output to preForth.asm
-preForth.asm: preForth.pre preForth-i386-backend.pre load-i386-preForth.fs
-	cat preForth-i386-rts.pre preForth-rts.pre preForth-i386-backend.pre preForth.pre \
-	    | $(HOSTFORTH) load-i386-preForth.fs >preForth.asm
-
-preForth: preForth.$(UNIXFLAVOUR)
-	cp preForth.$(UNIXFLAVOUR) preForth
-
-%.asm: %.pre preForth preForth-i386-rts.pre preForth-rts.pre
-	cat preForth-i386-rts.pre preForth-rts.pre $< | ./preForth >$@
-
-%: %.$(UNIXFLAVOUR)
-	cp $< $@
-
+preForth.asm: \
+preForth-i386-rts.pre \
+preForth-rts.pre \
+preForth-i386-backend.pre \
+preForth.pre \
+load-i386-preForth.fs
+	cat \
+preForth-i386-rts.pre \
+preForth-rts.pre \
+preForth-i386-backend.pre \
+preForth.pre \
+|$(HOSTFORTH) load-i386-preForth.fs >preForth.asm
+
+%.asm: %.pre preForth-i386-rts.pre preForth-rts.pre preForth
+	./preForth preForth-i386-rts.pre preForth-rts.pre $< >$@
+
+ifeq ($(UNIXFLAVOUR),Linux)
 # assemble and link executable on linux
-%.Linux: %.asm
+%: %.asm
 	fasm $< [email protected]
 	LDEMULATION=elf_i386 ld -arch i386 -o $@ \
-	   -dynamic-linker /lib32/ld-linux.so.2 \
-	   /usr/lib/i386-linux-gnu/crt1.o /usr/lib/i386-linux-gnu/crti.o \
-	   [email protected] \
-	   -lc /usr/lib/i386-linux-gnu/crtn.o
+-dynamic-linker /lib32/ld-linux.so.2 \
+/usr/lib/i386-linux-gnu/crt1.o /usr/lib/i386-linux-gnu/crti.o \
+[email protected] \
+-lc /usr/lib/i386-linux-gnu/crtn.o
 	# rm [email protected]
-
+else
+ifeq ($(UNIXFLAVOUR),Darwin)
 # assemble and link executable on MacOS
-%.Darwin: %.asm
+%: %.asm
 	fasm $< [email protected]
 	objconv -fmacho32 -nu [email protected] $@_m.o
 	ld -arch i386 -macosx_version_min 10.6 -o $@ \
-	  $@_m.o /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib/crt1.o /usr/lib/libc.dylib
+$@_m.o /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib/crt1.o \
+/usr/lib/libc.dylib
 	# rm [email protected] $@_m.o
+endif
+endif
 
 # run preForth on its own source code to perform a bootstrap 
 # should produce identical results
-bootstrap: preForth preForth-i386-backend.pre preForth.pre preForth.$(EXT)
-	cat preForth-i386-rts.pre preForth-rts.pre preForth-i386-backend.pre preForth.pre\
-           | ./preForth >preForth1.$(EXT)
+bootstrap: \
+preForth-i386-rts.pre \
+preForth-rts.pre \
+preForth-i386-backend.pre \
+preForth.pre \
+preForth \
+preForth.$(EXT)
+	./preForth \
+preForth-i386-rts.pre \
+preForth-rts.pre \
+preForth-i386-backend.pre \
+preForth.pre \
+>preForth1.$(EXT)
 	cmp preForth.$(EXT) preForth1.$(EXT)
 
 # preForth connected to stdin - output to stdout
 .PHONY=visible-bootstrap
 visible-bootstrap: preForth preForth-i386-backend.pre preForth.pre 
-	cat preForth-i386-backend.pre preForth.pre | ./preForth
+	./preForth preForth-i386-backend.pre preForth.pre
 
 # ------------------------------------------------------------------------
 # Docker support (for Linux version)
@@ -89,16 +108,48 @@ rundocker: docker-image
 # ------------------------------------------------------------------------
 # seedForth
 # ------------------------------------------------------------------------
-seedForth.$(EXT): seedForth-i386.pre preForth
-	cat seedForth-i386.pre | ./preForth >seedForth.$(EXT)
-
-seedForth: seedForth.$(UNIXFLAVOUR)
-	cp seedForth.$(UNIXFLAVOUR) seedForth
-
-%.seed: %.seedsource seedForth-tokenizer.fs
-	gforth seedForth-tokenizer.fs $<
-
+seedForth.$(EXT): \
+seedForth-i386-header.pre \
+preForth-i386-rts.pre \
+seedForth-i386.pre \
+seedForth.pre \
+preForth
+	./preForth \
+seedForth-i386-header.pre \
+preForth-i386-rts.pre \
+seedForth-i386.pre \
+seedForth.pre \
+>seedForth.$(EXT)
+
+# a little ugly, because gforth insists upon echoing anything read from
+# stdin, and also does not allow parsing words to cross a file boundary,
+# so we will concatenate the tokenizer and source into a *.fs file first
+seedForthDemo.seed: crc10.forth seedForth-tokenizer.fs seedForthDemo.seedsource
+	cat $^ >__temp__.fs
+	gforth __temp__.fs -e bye >$@
+	rm __temp__.fs
+
+seedForthBoot.seed: \
+crc10.forth \
+seedForth-tokenizer.fs \
+seedForthRuntime.seedsource \
+seedForthBoot.seedsource
+	cat $^ >__temp__.fs
+	gforth __temp__.fs -e bye >$@
+	rm __temp__.fs
+
+seedForthInteractive.seed: \
+crc10.forth \
+seedForth-tokenizer.fs \
+seedForthRuntime.seedsource \
+seedForthInteractive.seedsource
+	cat $^ >__temp__.fs
+	gforth __temp__.fs -e bye >$@
+	rm __temp__.fs
+
+crc10.forth: crc10_gen.forth
+	gforth $^ -e bye >$@
 
 .PHONY=clean
 clean:
-	rm -f *.asm *.o *.fas *.s *.c *.Darwin *.Linux preForthdemo preForth forth seedForth seedForthDemo.seed seedForthInteractive.seed
+	rm -f *.asm *.o *.seed preForthdemo preForth seedForth __temp__.fs

preForth/crc10_gen.forth

@@ -0,0 +1,50 @@
+\ run this as follows:
+\ ./seedForth seedForthBoot.seed runtime.forth crc10_gen.forth >crc10.forth
+
+817 Constant poly \ CRC-10/ATM, truncated polynomial 0x233, bit reversed 0x331
+
+\ we don't want MS-DOS style line endings in the generated CRC table file
+: lf 10 emit ;
+
+: gen
+  ." Create crc10_tab" lf
+  256 0 ?DO
+    I 8 0 ?DO
+      \ split into LSB and the remaining bits
+      \ in seedForth: 2 u/mod swap
+      \ in gForth: 2 /mod swap
+      dup 1 rshift swap 1 and
+
+      IF poly xor THEN
+    LOOP
+    . ',' emit
+    I 7 and 7 = IF lf ELSE space THEN
+  LOOP
+  lf
+  ." : crc10 ( addr len crc -- crc )" lf
+  ."   swap 0 ?DO ( addr crc )" lf
+  ."     \ retrieve next character" lf
+  ."     over I + c@" lf
+  lf
+  ."     \ xor into low bits of crc" lf
+  ."     xor" lf
+  lf
+  ."     \ separate into table index and remaining bits" lf
+  ."     \ 256 u/mod swap" lf
+  ."     dup 8 rshift swap 255 and" lf
+  lf
+  ."     \ look up new bits from table" lf
+  ."     cells crc10_tab + @" lf
+  lf
+  ."     \ combine with remaining (shifted right) bits" lf
+  ."     xor" lf
+  ."   LOOP" lf
+  ."   nip" lf
+  ." ;" lf
+  lf
+  ." \ testing:" lf
+  ." \ Create hello 104 c, 101 c, 108 c, 108 c, 111 c," lf
+  ." \ hello 5 1023 crc10 . ;" lf
+;
+
+gen