feat: nanoGPT implementation using Reactant

.gitignore

@@ -44,3 +44,5 @@ benchmarks/results
 # Generated by tutorials
 pinn_nested_ad.gif
 *.mlir
+
+*.jld2

examples/NanoGPT/Project.toml

@@ -0,0 +1,33 @@
+[deps]
+BytePairEncoding = "a4280ba5-8788-555a-8ca8-4a8c3d966a71"
+Comonicon = "863f3e99-da2a-4334-8734-de3dacbe5542"
+ConcreteStructs = "2569d6c7-a4a2-43d3-a901-331e8e4be471"
+DataDeps = "124859b0-ceae-595e-8997-d05f6a7a8dfe"
+Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9"
+JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
+Lux = "b2108857-7c20-44ae-9111-449ecde12c47"
+MLUtils = "f1d291b0-491e-4a28-83b9-f70985020b54"
+NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
+OneHotArrays = "0b1bfda6-eb8a-41d2-88d8-f5af5cad476f"
+Optimisers = "3bd65402-5787-11e9-1adc-39752487f4e2"
+Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+Reactant = "3c362404-f566-11ee-1572-e11a4b42c853"
+Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
+StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
+
+[compat]
+Comonicon = "1"
+DataDeps = "0.7"
+Enzyme = "0.13.14"
+JLD2 = "0.5"
+Lux = "1.2.3"
+MLUtils = "0.4"
+NNlib = "0.9.24"
+OneHotArrays = "0.2.5"
+Optimisers = "0.4.1"
+Printf = "1.10"
+Random = "1.10"
+Reactant = "0.2.5"
+Statistics = "1.10"
+StatsBase = "0.34.3"

examples/NanoGPT/README.md

@@ -0,0 +1,58 @@
+# NanoGPT using Lux & Reactant
+
+## Requirements
+
+* Install [julia](https://julialang.org/)
+* In the Julia REPL instantiate the `Project.toml` in the parent directory
+
+## Training
+
+To train a model, run `main.jl` with the necessary parameters.
+
+```bash
+julia --startup=no --project=examples/NanoGPT --threads=auto examples/NanoGPT/main.jl
+```
+
+## Inference
+
+To run inference on a trained model, run `main.jl` with the necessary parameters.
+
+```bash
+julia --startup=no --project=examples/NanoGPT --threads=auto examples/NanoGPT/main.jl \
+  --inference \
+  --model-path=<path to model checkpoint>
+```
+
+## Usage
+
+```bash
+  main
+
+Usage
+
+  main [options] [flags]
+
+Options
+
+  --n-embed <64::Int>
+  --n-hidden <256::Int>
+  --n-heads <4::Int>
+  --qk-dim <16::Int>
+  --v-dim <16::Int>
+  --n-layers <6::Int>
+  --sequence-length <64::Int>
+  --batchsize <128::Int>
+  --dropout-rate <0.0::Float32>
+  --test-split <0.1::Float64>
+  --lr <0.01::Float64>
+  --epochs <100::Int>
+  --model-path <::String>
+  --seed <::Union{String, Vector{String}}>
+  --output-length <1024::Int>
+
+Flags
+
+  --inference
+  -h, --help                                                Print this help message.
+  --version                                                 Print version.
+```