diff --git a/asdf.fish b/asdf.fish
index 44ac524a..172b1f63 100644
--- a/asdf.fish
+++ b/asdf.fish
@@ -5,4 +5,7 @@ set -l asdf_dir (dirname (status -f))
 # we get an ugly warning when setting the path if shims does not exist
 mkdir -p $asdf_dir/shims
 
-set -xg PATH $asdf_dir/bin $asdf_dir/shims $PATH
+if not contains $asdf_dir/bin $asdf_dir/shims $PATH
+and test -d $asdf/bin and test -d $asdf/shims
+  set -xg PATH $asdf_dir/bin $asdf_dir/shims $PATH
+end