diff --git a/lua/oil/init.lua b/lua/oil/init.lua
index 871c4f6..da2ecf8 100644
--- a/lua/oil/init.lua
+++ b/lua/oil/init.lua
@@ -639,13 +639,10 @@ M.select = function(opts, callback)
         mods = mods,
       })
 
-      if not opts.preview and preview_win and entry_is_file then
-        vim.api.nvim_win_close(preview_win, true)
-      end
-
       if opts.preview then
         vim.api.nvim_set_option_value("previewwindow", true, { scope = "local", win = 0 })
         vim.w.oil_entry_id = entry.id
+        vim.w.oil_source_win = prev_win
         vim.api.nvim_set_current_win(prev_win)
       end
       open_next_entry(cb)
@@ -910,6 +907,25 @@ local function load_oil_buffer(bufnr)
   adapter.normalize_url(bufname, finish)
 end
 
+local function close_preview_window_if_not_in_oil()
+  local util = require("oil.util")
+  local preview_win_id = util.get_preview_win()
+  if not preview_win_id or not vim.w[preview_win_id].oil_entry_id then
+    return
+  end
+
+  local oil_source_win = vim.w[preview_win_id].oil_source_win
+  if oil_source_win and vim.api.nvim_win_is_valid(oil_source_win) then
+    local src_buf = vim.api.nvim_win_get_buf(oil_source_win)
+    if util.is_oil_bufnr(src_buf) then
+      return
+    end
+  end
+
+  -- This can fail if it's the last window open
+  pcall(vim.api.nvim_win_close, preview_win_id, true)
+end
+
 ---Initialize oil
 ---@param opts nil|table
 M.setup = function(opts)
@@ -1072,6 +1088,8 @@ M.setup = function(opts)
         -- Oil buffers have to run it in BufReadCmd after confirming they are a directory or a file
         restore_alt_buf()
       end
+
+      close_preview_window_if_not_in_oil()
     end,
   })