diff --git a/.cirrus.yml b/.cirrus.yml
index a1264ca608..92507f6d50 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -26,3 +26,17 @@ freebsd_task:
     - sudo -u cirrus gmake unittest
   oldtest_script:
     - sudo -u cirrus gmake oldtest
+
+with_external_deps_task:
+  container:
+    dockerfile: ci/Dockerfile.external_deps
+  deps_script:
+      # Ideally all dependencies should external for this job, but some
+      # dependencies don't have the required version available. We use the
+      # bundled versions for these with the hopes of being able to remove them
+      # later on.
+    - cmake -S cmake.deps -B .deps -G Ninja -D USE_BUNDLED=OFF -D USE_BUNDLED_LIBVTERM=ON -D USE_BUNDLED_TS=ON
+    - cmake --build .deps
+  build_script:
+    - cmake --preset ci
+    - cmake --build build
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 2fc6b13ec4..2b0b9ff20a 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -353,47 +353,3 @@ jobs:
         run: |
           cd test/old/testdir
           mingw32-make VERBOSE=1
-
-  with-external-deps:
-    runs-on: ubuntu-22.04
-    timeout-minutes: 10
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Install dependencies
-        run: |
-          sudo add-apt-repository ppa:neovim-ppa/stable
-          ./.github/scripts/install_deps.sh
-          sudo apt-get install -y \
-            libluajit-5.1-dev \
-            libmsgpack-dev \
-            libtermkey-dev \
-            libunibilium-dev \
-            libuv1-dev \
-            lua-filesystem \
-            lua-lpeg \
-            luajit \
-            lua-luv-dev
-            # libtree-sitter-dev \
-            # libvterm-dev
-
-            # Remove comments from packages once we start using these external
-            # dependencies.
-
-      - uses: ./.github/actions/cache
-
-      - name: Build third-party deps
-        run: |
-          # Ideally all dependencies should external for this job, but some
-          # dependencies don't have the required version available. We use the
-          # bundled versions for these with the hopes of being able to remove them
-          # later on.
-          cmake -S cmake.deps -B .deps -G Ninja -D USE_BUNDLED=OFF \
-            -D USE_BUNDLED_LIBVTERM=ON \
-            -D USE_BUNDLED_TS=ON
-          cmake --build .deps
-
-      - name: Build
-        run: |
-          cmake --preset ci
-          cmake --build build
diff --git a/ci/Dockerfile.external_deps b/ci/Dockerfile.external_deps
new file mode 100644
index 0000000000..0d2a55dd19
--- /dev/null
+++ b/ci/Dockerfile.external_deps
@@ -0,0 +1,22 @@
+FROM ubuntu:22.04
+
+RUN apt-get update && \
+    apt-get install -y software-properties-common && \
+    add-apt-repository ppa:neovim-ppa/stable && \
+    apt-get install -y ninja-build \
+            gettext \
+            cmake \
+            unzip \
+            curl \
+            git \
+            libluajit-5.1-dev \
+            libmsgpack-dev \
+            libtermkey-dev \
+            libunibilium-dev \
+            libuv1-dev \
+            lua-filesystem \
+            lua-lpeg \
+            luajit \
+            lua-luv-dev \
+            libtree-sitter-dev \
+            libvterm-dev