For PopUp using Lightning Web Component, check https://www.infallibletechie.com/2020/03/popup-using-lightning-web-component-lwc.html. Following example is for Salesforce classic only.

1. Add the below .zip file in Static Resources.

Click this link to download the file jQueryForPopup.

2. Create a Custom button with On Click Java script functionality.

Java script code for button:

 {!REQUIRESCRIPT('/soap/ajax/26.0/connection.js')}
{!REQUIRESCRIPT('/js/functions.js')}
{!REQUIRESCRIPT('/resource/jQueryForPopup/jQuery/jquery-1.8.2.min.js')}
{!REQUIRESCRIPT('/resource/jQueryForPopup/jQuery/ui/jquery-ui-1.9.1.custom.min.js')}
{!REQUIRESCRIPT('/resource/jQueryForPopup/jQuery/postmessage/jquery.ba-postmessage.js')}
{!REQUIRESCRIPT('/resource/jQueryForPopup/jQuery/bbq/jquery.ba-bbq.min.js')}
requireCssFile('/resource/jQueryForPopup/jQuery/ui/css/ui-lightness/jquery-ui-1.9.1.custom.min.css');
function requireCssFile(filename)
{
var fileref = document.createElement('link');
fileref.setAttribute('rel', 'stylesheet');
fileref.setAttribute('type', 'text/css');
fileref.setAttribute('href', filename);
document.getElementsByTagName('head')[0].appendChild(fileref);
}
var j$ = jQuery.noConflict();
var iframe_url = '{!URLFOR("/apex/Sample")}';
var child_domain = iframe_url.substring(0, iframe_url.indexOf('/', 9));
var parent_domain = window.location.protocol + '//' + window.location.host;
var j$modalDialog = j$('<div id="opppopup"></div>')
.html('<iframe id="iframeContentId" src="' + iframe_url + '" frameborder="0" height="100%" width="100%" marginheight="0" marginwidth="0" scrolling="no" />')
.dialog({
autoOpen: false,
title: 'Sample Popup',
resizable: true,
width: 800,
height: 540,
autoResize: true,
modal: true,
draggable: true
});

j$modalDialog.dialog('open');

3. Add the Custom button created to the page layout and check the popup.