diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 204b95cb..43826292 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -4,6 +4,10 @@ on:
     branches: [master]
   pull_request:
     types: [opened, synchronize, reopened]
+# needed to allow julia-actions/cache to delete old caches that it has created
+permissions:
+  actions: write
+  contents: read
 jobs:
   test:
     name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }}
@@ -35,3 +39,20 @@ jobs:
         with:
           file: lcov.info
           token: ${{ secrets.CODECOV_TOKEN }}
+  benchmark:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - file: 'run_benchmarks.jl'
+          - file: 'run_benchmarks_jump.jl'
+          - file: 'run_benchmarks_pmedian.jl'
+          - file: 'run_benders_quantile_regression_benchmark.jl'
+    steps:
+      - uses: actions/checkout@v4
+      - uses: julia-actions/setup-julia@latest
+        with:
+          version: '1'
+      - run: julia --project=benchmark -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()'
+      - run: julia --color=yes --project=benchmark benchmark/${{ matrix.file }}
diff --git a/.gitignore b/.gitignore
index a2a193cd..9390fdf8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,4 +8,4 @@ Manifest.toml
 test/MOItests.jl
 test/quadratic_tests.jl
 *.json
-*.txt
\ No newline at end of file
+*.txt
diff --git a/benchmark/.gitignore b/benchmark/.gitignore
deleted file mode 100644
index 05e9b4cc..00000000
--- a/benchmark/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-!Manifest.toml
\ No newline at end of file
diff --git a/benchmark/Project.toml b/benchmark/Project.toml
index 124b3824..5890cd19 100644
--- a/benchmark/Project.toml
+++ b/benchmark/Project.toml
@@ -7,3 +7,12 @@ ParameterJuMP = "774612a8-9878-5177-865a-ca53ae2495f9"
 ParametricOptInterface = "0ce4ce61-57bf-432b-a095-efac525d185e"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 TimerOutputs = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f"
+
+[compat]
+BenchmarkTools = "1"
+HiGHS = "1"
+JuMP = "1"
+MathOptInterface = "1"
+ParameterJuMP = "0.4"
+Random = "1"
+TimerOutputs = "0.5"