diff --git a/action.yaml b/action.yaml
index f69e11f..05d9a1f 100644
--- a/action.yaml
+++ b/action.yaml
@@ -33,6 +33,11 @@ inputs:
     default: ''
     description: Content of `~/.ssh/known_hosts` file.
 
+  disable-strict-host-checking:
+    required: false
+    default: 'true'
+    description: Disable Strict Host Checking if no known_hosts are provided
+
   ssh-config:
     required: false
     default: ''
@@ -71,6 +76,7 @@ inputs:
 runs:
   using: 'node20'
   main: 'index.js'
+  post: 'cleanup.js'
 
 branding:
   color: blue
diff --git a/cleanup.js b/cleanup.js
new file mode 100644
index 0000000..2ae0493
--- /dev/null
+++ b/cleanup.js
@@ -0,0 +1,22 @@
+import core from '@actions/core'
+import { $ } from 'zx'
+
+void (async function main() {
+  try {
+    await cleanup()
+  } catch (err) {
+    core.setFailed(err.message)
+  }
+})()
+
+async function cleanup() {
+  if (core.getBooleanInput('skip-ssh-setup')) {
+    return
+  }
+
+  const sshAgentPid = core.getState('ssh-agent-pid')
+
+  // Remove all keys from ssh-agent and kill process
+  await $`ssh-add -D`
+  await $`kill ${sshAgentPid}`
+}
diff --git a/index.js b/index.js
index b50cc0b..9655fab 100644
--- a/index.js
+++ b/index.js
@@ -15,15 +15,35 @@ async function ssh() {
     return
   }
 
-  let sshHomeDir = `${process.env['HOME']}/.ssh`
+  const sshHomeDir = `${process.env['HOME']}/.ssh`
 
   if (!fs.existsSync(sshHomeDir)) {
     fs.mkdirSync(sshHomeDir)
   }
 
-  let authSock = '/tmp/ssh-auth.sock'
-  await $`ssh-agent -a ${authSock}`
-  core.exportVariable('SSH_AUTH_SOCK', authSock)
+  // Unfortunately running the output into bash or eval-ing it does
+  // not persist the exported environment variables, so instead we
+  // parse out the variables via regex, not ideal but works a treat.
+  const sshAgentOutput = await $`ssh-agent`
+
+  const sshAgentSocket = sshAgentOutput
+    .stdout
+    .match(/SSH_AUTH_SOCK=(?<path>.*); export SSH_AUTH_SOCK;/)
+    ?.groups['path'] ?? null;
+
+  const sshAgentProcessId = sshAgentOutput
+    .stdout
+    .match(/SSH_AGENT_PID=(?<pid>\d+); export SSH_AGENT_PID;/)
+    ?.groups['pid'] ?? null;
+
+  if (!sshAgentSocket || !sshAgentProcessId) {
+    throw new Error('Failed to start ssh-agent')
+  }
+
+  core.exportVariable('SSH_AUTH_SOCK', sshAgentSocket.trim())
+  core.exportVariable('SSH_AGENT_PID', sshAgentProcessId.trim())
+
+  core.saveState('ssh-agent-pid', sshAgentProcessId.trim())
 
   let privateKey = core.getInput('private-key')
   if (privateKey !== '') {
@@ -39,8 +59,10 @@ async function ssh() {
     fs.appendFileSync(`${sshHomeDir}/known_hosts`, knownHosts)
     fs.chmodSync(`${sshHomeDir}/known_hosts`, '600')
   } else {
-    fs.appendFileSync(`${sshHomeDir}/config`, `StrictHostKeyChecking no`)
-    fs.chmodSync(`${sshHomeDir}/config`, '600')
+    if (core.getBooleanInput('disable-strict-host-checking')) {
+      fs.appendFileSync(`${sshHomeDir}/config`, `StrictHostKeyChecking no`)
+      fs.chmodSync(`${sshHomeDir}/config`, '600')
+    }
   }
 
   let sshConfig = core.getInput('ssh-config')