feat: nanoGPT implementation using Reactant

.gitignore

@@ -53,3 +53,4 @@ poetry.lock
 .venv
 .python-version
 __pycache__
+*.jld2

examples/NanoGPT/Project.toml

@@ -0,0 +1,30 @@
+[deps]
+ArgParse = "c7e460c6-2fb9-53a9-8c5b-16f535851c63"
+BytePairEncoding = "a4280ba5-8788-555a-8ca8-4a8c3d966a71"
+ConcreteStructs = "2569d6c7-a4a2-43d3-a901-331e8e4be471"
+DataDeps = "124859b0-ceae-595e-8997-d05f6a7a8dfe"
+Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9"
+JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
+Lux = "b2108857-7c20-44ae-9111-449ecde12c47"
+MLUtils = "f1d291b0-491e-4a28-83b9-f70985020b54"
+NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
+OneHotArrays = "0b1bfda6-eb8a-41d2-88d8-f5af5cad476f"
+Optimisers = "3bd65402-5787-11e9-1adc-39752487f4e2"
+Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+Reactant = "3c362404-f566-11ee-1572-e11a4b42c853"
+Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
+
+[compat]
+DataDeps = "0.7"
+Enzyme = "0.13.14"
+JLD2 = "0.5, 0.6"
+Lux = "1.21"
+MLUtils = "0.4"
+NNlib = "0.9.24"
+OneHotArrays = "0.2.5"
+Optimisers = "0.4.1"
+Printf = "1.10"
+Random = "1.10"
+Reactant = "0.2.5"
+Statistics = "1.10"

examples/NanoGPT/README.md

@@ -0,0 +1,58 @@
+# NanoGPT using Lux & Reactant
+
+## Requirements
+
+* Install [julia](https://julialang.org/)
+* In the Julia REPL instantiate the `Project.toml` in the parent directory
+
+## Training
+
+To train a model, run `main.jl` with the necessary parameters.
+
+```bash
+julia --startup=no --project=examples/NanoGPT --threads=auto examples/NanoGPT/main.jl
+```
+
+## Inference
+
+To run inference on a trained model, run `main.jl` with the necessary parameters.
+
+```bash
+julia --startup=no --project=examples/NanoGPT --threads=auto examples/NanoGPT/main.jl \
+  --inference \
+  --model-path=<path to model checkpoint>
+```
+
+## Usage
+
+```bash
+  main
+
+Usage
+
+  main [options] [flags]
+
+Options
+
+  --n-embed <64::Int>
+  --n-hidden <256::Int>
+  --n-heads <4::Int>
+  --qk-dim <16::Int>
+  --v-dim <16::Int>
+  --n-layers <6::Int>
+  --sequence-length <64::Int>
+  --batchsize <128::Int>
+  --dropout-rate <0.0::Float32>
+  --test-split <0.1::Float64>
+  --lr <0.01::Float64>
+  --epochs <100::Int>
+  --model-path <::String>
+  --seed <::Union{String, Vector{String}}>
+  --output-length <1024::Int>
+
+Flags
+
+  --inference
+  -h, --help                                                Print this help message.
+  --version                                                 Print version.
+```