diff --git a/lua/oil/init.lua b/lua/oil/init.lua
index dbf5296..3f4c414 100644
--- a/lua/oil/init.lua
+++ b/lua/oil/init.lua
@@ -1646,7 +1646,9 @@ M.setup = function(opts)
       pattern = '*',
       nested = true,
       callback = function(params)
-        maybe_hijack_directory_buffer(params.buf)
+        if maybe_hijack_directory_buffer(params.buf) and vim.v.vim_did_enter == 1 then
+          M.load_oil_buffer(params.buf)
+        end
       end,
     })