Python Forum
Running headless selenium with proxy - Printable Version

+- Python Forum (https://python-forum.io)
+-- Forum: Python Coding (https://python-forum.io/forum-7.html)
+--- Forum: Networking (https://python-forum.io/forum-12.html)
+--- Thread: Running headless selenium with proxy (/thread-25040.html)



Running headless selenium with proxy - julio2000 - Mar-16-2020

So I've created this code wich allows me to run selenium with a proxy (IP:PORT:AUTH:PASS)
import os
import zipfile
import time
from selenium import webdriver

PROXY_HOST = 'My host is here(dont show it because it is private)' 
PROXY_PORT = 7777
PROXY_USER = 'my username(dont show it because it is private)'
PROXY_PASS = 'the password(dont show it because it is private)'


manifest_json = """
{
    "version": "1.0.0",
    "manifest_version": 2,
    "name": "Chrome Proxy",
    "permissions": [
        "proxy",
        "tabs",
        "unlimitedStorage",
        "storage",
        "<all_urls>",
        "webRequest",
        "webRequestBlocking"
    ],
    "background": {
        "scripts": ["background.js"]
    },
    "minimum_chrome_version":"22.0.0"
}
"""

background_js = """
var config = {
        mode: "fixed_servers",
        rules: {
          singleProxy: {
            scheme: "http",
            host: "%s",
            port: parseInt(%s)
          },
          bypassList: ["localhost"]
        }
      };

chrome.proxy.settings.set({value: config, scope: "regular"}, function() {});

function callbackFn(details) {
    return {
        authCredentials: {
            username: "%s",
            password: "%s"
        }
    };
}

chrome.webRequest.onAuthRequired.addListener(
            callbackFn,
            {urls: ["<all_urls>"]},
            ['blocking']
);
""" % (PROXY_HOST, PROXY_PORT, PROXY_USER, PROXY_PASS)


def get_chromedriver(use_proxy=False, user_agent=None):
    path = os.path.dirname(os.path.abspath(__file__))
    chrome_options = webdriver.ChromeOptions()
    if use_proxy:
        pluginfile = 'proxy_auth_plugin.zip'

        with zipfile.ZipFile(pluginfile, 'w') as zp:
            zp.writestr("manifest.json", manifest_json)
            zp.writestr("background.js", background_js)
        chrome_options.add_extension(pluginfile)
    if user_agent:
        chrome_options.add_argument('--user-agent=%s' % user_agent)
    driver = webdriver.Chrome(
        os.path.join(path, 'chromedriver'),
        chrome_options=chrome_options)
    return driver

def main():
    driver = get_chromedriver(use_proxy=True)
    #driver.get('https://www.google.com/search?q=my+ip+address')
    driver.get('https://whatismyipaddress.com/')
    time.sleep(100)

if __name__ == '__main__':
    main()
So when I run this, everything works fine and the proxy works. But as soon as I run this headless (wich is my goal) it gives the following error:
Error:
C:/Users/Julius/PycharmProjects/Fenix/venv/bla.py:96: DeprecationWarning: use options instead of chrome_options driver = webdriver.Chrome( Traceback (most recent call last): File "C:/Users/Julius/PycharmProjects/Fenix/venv/bla.py", line 108, in <module> main() File "C:/Users/Julius/PycharmProjects/Fenix/venv/bla.py", line 102, in main driver = get_chromedriver(use_proxy=True) File "C:/Users/Julius/PycharmProjects/Fenix/venv/bla.py", line 96, in get_chromedriver driver = webdriver.Chrome( File "C:\Users\Julius\AppData\Local\Programs\Python\Python38-32\lib\site-packages\selenium\webdriver\chrome\webdriver.py", line 76, in __init__ RemoteWebDriver.__init__( File "C:\Users\Julius\AppData\Local\Programs\Python\Python38-32\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 157, in __init__ self.start_session(capabilities, browser_profile) File "C:\Users\Julius\AppData\Local\Programs\Python\Python38-32\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 252, in start_session response = self.execute(Command.NEW_SESSION, parameters) File "C:\Users\Julius\AppData\Local\Programs\Python\Python38-32\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 321, in execute self.error_handler.check_response(response) File "C:\Users\Julius\AppData\Local\Programs\Python\Python38-32\lib\site-packages\selenium\webdriver\remote\errorhandler.py", line 242, in check_response raise exception_class(message, screen, stacktrace) selenium.common.exceptions.WebDriverException: Message: unknown error: failed to wait for extension background page to load: chrome-extension://hppjijlamkeconblfhjmcjlopacliakh/_generated_background_page.html from unknown error: page could not be found: chrome-extension://hppjijlamkeconblfhjmcjlopacliakh/_generated_background_page.html
does someone know the fix for this? Or a different method to run selenium headless with a proxy?
please let me know!!


RE: Running headless selenium with proxy - Larz60+ - Mar-17-2020

line 96 didn't allow enough time for process to finish. However, line numbers don't match code.
I suspect line 77 is the error item.
try adding a delat of a couple of seconds after line 79
    time.sleep(2)
you can shorten delay by experimenting