diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
index c031682a..f2c310aa 100644
--- a/.github/workflows/workflow.yml
+++ b/.github/workflows/workflow.yml
@@ -28,10 +28,16 @@ jobs:
     steps:
     - name: Checkout
       uses: actions/checkout@v2
+    - name: Clear tool cache
+      run: mv "${{ runner.tool_cache }}" "${{ runner.tool_cache }}.old"
+    - name: Clear local maven repository
+      run: mv "~/.m2/repository" "~/.m2/repository.bak" && mkdir ~/.m2/repository
     - name: Setup java 7
       uses: ./
       with:
-        java-version: 7.x
+        java-version: 7.0.242
+    - name: Verify java 7
+      run: __tests__/verify-java.sh
     - run: |
         which java
         java -version
diff --git a/__tests__/verify-java.sh b/__tests__/verify-java.sh
new file mode 100755
index 00000000..b27f0d08
--- /dev/null
+++ b/__tests__/verify-java.sh
@@ -0,0 +1,25 @@
+#!/bin/sh
+
+if [ -z "$1" ]; then
+  echo "Must supply java version argument"
+  exit 1
+fi
+
+java_version="$(java -version)"
+echo "Found java version: $node_version"
+if [ -z "$(echo $node_version | grep --fixed-strings $1)" ]; then
+  echo "Unexpected version"
+  exit 1
+fi
+
+echo "Building sample proj"
+proj_dir=__tests/sample-proj
+mvn --file $proj_dir clean install dependency:copy-dependencies || exit 1
+
+echo "Testing compiled app"
+sample_output="$(java --class-path $proj_dir/target/HelloWorld-1.0-SNAPSHOT.jar:$proj_dir/target/dependency/json-20190722.jar com.helloworld.App)"
+echo "Sample output: $sample_output"
+if [ -z "$(echo $sample_output | grep --fixed-strings '{"Hello":"World!"}')" ]; then
+  echo "Unexpected output"
+  exit 1
+fi