Python Forum

Full Version: Flask with paramiko based ssh client gives gevent LoopExit exception
You're currently viewing a stripped down version of our content. View the full version with proper formatting.
I am getting gevent LoopExit error when using paramiko based ssh client with flask server.

Sample code to reproduce the problem.
#!usr/bin/python3.6
#from gevent import monkey; monkey.patch_all()
from pssh.clients.native import SSHClient
import flask
from flask import request

app = flask.Flask(__name__)
app.config["DEBUG"] = False

def checkcpu(ip):
    cmd_cpu_usage = '''awk '{u=$2+$4; t=$2+$4+$5; if (NR==1){u1=u; t1=t;} else print ($2+$4-u1) * 100 / (t-t1) "%"; }' <(grep 'cpu ' /proc/stat) <(sleep 1;grep 'cpu ' /proc/stat)'''
    server = SSHClient(ip,user='user',password='password')
    output = server.run_command(cmd_cpu_usage)
    status = ''
    for line in output[2]:
        status += line
    return status
 
@app.route('/', methods=['GET'])
def home():
    return '<h1>_WORKS_<h1>' 

@app.route('/v1/check/cpu', methods=['GET'])
def api_cpu_check():
    if 'ip' in request.args:
        ip = request.args['ip']
    else:
        return "Error: IP required"
    return checkcpu(ip)
Error:
ERROR in app: Exception on /v1/check/cpu [GET] Traceback (most recent call last): File "/usr/local/lib/python3.6/dist-packages/flask/app.py", line 2292, in wsgi_app response = self.full_dispatch_request() File "/usr/local/lib/python3.6/dist-packages/flask/app.py", line 1815, in full_dispatch_request rv = self.handle_user_exception(e) File "/usr/local/lib/python3.6/dist-packages/flask/app.py", line 1718, in handle_user_exception reraise(exc_type, exc_value, tb) File "/usr/local/lib/python3.6/dist-packages/flask/_compat.py", line 35, in reraise raise value File "/usr/local/lib/python3.6/dist-packages/flask/app.py", line 1813, in full_dispatch_request rv = self.dispatch_request() File "/usr/local/lib/python3.6/dist-packages/flask/app.py", line 1799, in dispatch_request return self.view_functions[rule.endpoint](**req.view_args) File "/var/www/az/test.py", line 33, in api_cpu_check return checkcpu(ip) File "/var/www/az/test.py", line 16, in checkcpu server = SSHClient(ip,user='user',password='*********') File "/usr/local/lib/python3.6/dist-packages/pssh/clients/native/single.py", line 130, in __init__ self._connect(self._host, self.port) File "/usr/local/lib/python3.6/dist-packages/pssh/clients/native/single.py", line 209, in _connect self.sock.connect((host, port)) File "/usr/local/lib/python3.6/dist-packages/gevent/_socket3.py", line 329, in connect address = _socketcommon._resolve_addr(self._sock, address) File "/usr/local/lib/python3.6/dist-packages/gevent/_socketcommon.py", line 376, in _resolve_addr r = getaddrinfo(host, None, sock.family) File "/usr/local/lib/python3.6/dist-packages/gevent/_socketcommon.py", line 208, in getaddrinfo return get_hub().resolver.getaddrinfo(host, port, family, type, proto, flags) File "/usr/local/lib/python3.6/dist-packages/gevent/resolver/thread.py", line 65, in getaddrinfo return self.pool.apply(_socket.getaddrinfo, args, kwargs) File "/usr/local/lib/python3.6/dist-packages/gevent/pool.py", line 159, in apply return self.spawn(func, *args, **kwds).get() File "src/gevent/event.py", line 381, in gevent._event.AsyncResult.get File "src/gevent/event.py", line 406, in gevent._event.AsyncResult.get File "src/gevent/event.py", line 117, in gevent._event._AbstractLinkable._wait_core File "src/gevent/event.py", line 119, in gevent._event._AbstractLinkable._wait_core File "src/gevent/_greenlet_primitives.py", line 59, in gevent.__greenlet_primitives.SwitchOutGreenletWithLoop.switch File "src/gevent/_greenlet_primitives.py", line 59, in gevent.__greenlet_primitives.SwitchOutGreenletWithLoop.switch File "src/gevent/_greenlet_primitives.py", line 63, in gevent.__greenlet_primitives.SwitchOutGreenletWithLoop.switch File "src/gevent/__greenlet_primitives.pxd", line 35, in gevent.__greenlet_primitives._greenlet_switch gevent.exceptions.LoopExit: This operation would block forever \tHub: <Hub '' at 0x7f929d9a3828 epoll default pending=0 ref=0 fileno=18 resolver=<gevent.resolver_thread.Resolver at 0x7f9298eeacf8 pool=<ThreadPool at 0x7f929b9ae1d0 0/1/10 hub=<Hub at 0x7f929d9a3828 thread_ident=0x140268143462272>>> threadpool=<ThreadPool at 0x7f929b9ae1d0 0/1/10 hub=<Hub at 0x7f929d9a3828 thread_ident=0x140268143462272>> thread_ident=0x7f92b8dd4780> \tHandles: []
This code works on development server on windows...But gives the above error with when run on linux server as wsgi application.

This didn't work for me, getting exact same error. Even tried to do monkeypatching in wsgi file like:
#!/usr/bin/python3.6
from gevent import monkey; monkey.patch_all()
import sys
sys.path.insert(0,"/var/www/testapp/")
from test import app as application
I am not familiar with gevent.

Any help will be appreciated.
The error message says that it fails on the line server = SSHClient(ip,user='user',password='*********') because in the end it may block forever. I don't know this pssh module, but using a timeout= keyword argument in SSHClient() may help solve the issue by preventing the call to block forever.
Thanks for replying.

I tried to minimize the waits, making few assumptions with following code:
  
server = SSHClient(ip,user='user',password='********',timeout=5,num_retries=2,retry_delay=1,keepalive_seconds=0)

#timeout : ssh session timeout
#num_retries: connection attempts
#keepalive_seconds: time interval for keep alive packet to be sent.
I have no idea about gevent but this should put a limit on blocks/waits
But same error persists.
Changed code to directly use paramiko.

#!usr/bin/python3.6
#from gevent import monkey; monkey.patch_all()
import flask
from flask import request
import paramiko

app = flask.Flask(__name__)
app.config["DEBUG"] = False

def exucute_cmd(cmd,ip,user,pw,port=22):
    client = paramiko.SSHClient()
    client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
    client.connect(ip, port, user, pw)
    shell = client.invoke_shell()
    _, stdout, stderr = client.exec_command(cmd)
    out=''
    for line in stdout.readlines():
        out+=line
    if not out:
        for line in stderr.readlines():
            out+=line

    shell.close()
    client.close()
    return out

def checkcpu(ip):
    cmd_cpu_usage = '''awk '{u=$2+$4; t=$2+$4+$5; if (NR==1){u1=u; t1=t;} else print ($2+$4-u1) * 100 / (t-t1) "%"; }' <(grep 'cpu ' /proc/stat) <(sleep 1;grep 'cpu ' /proc/stat)'''
    return exucute_cmd(cmd_cpu_usage,ip,'user','*******')
Getting Error on Paramiko Import:

Error:
Traceback (most recent call last): File "src/gevent/_waiter.py", line 116, in gevent.__waiter.Waiter.switch AssertionError: Can only use Waiter.switch method from the Hub greenlet 2018-12-25T07:30:17Z <io at 0x7f929b9716c8 fd=24 events=READ active callback=<bound method Waiter.switch of <gevent.__waiter.Waiter object at 0x7f929b98df98>> args=(<gevent.__waiter.Waiter object at 0x7f929b98df98>,)> failed with AssertionError mod_wsgi (pid=6149): Failed to exec Python script file '/var/www/az/az.wsgi'. mod_wsgi (pid=6149): Exception occurred processing WSGI script '/var/www/az/az.wsgi'. Traceback (most recent call last): File "/var/www/az/az.wsgi", line 8, in <module> from test import app as application File "/var/www/az/test.py", line 4, in <module> import paramiko File "/usr/local/lib/python3.6/dist-packages/paramiko/__init__.py", line 22, in <module> from paramiko.transport import SecurityOptions, Transport File "/usr/local/lib/python3.6/dist-packages/paramiko/transport.py", line 89, in <module> from paramiko.dsskey import DSSKey File "/usr/local/lib/python3.6/dist-packages/paramiko/dsskey.py", line 27, in <module> from cryptography.hazmat.primitives.asymmetric.utils import ( File "/usr/local/lib/python3.6/dist-packages/cryptography/hazmat/primitives/asymmetric/utils.py", line 9, in <module> from asn1crypto.algos import DSASignature File "/usr/local/lib/python3.6/dist-packages/asn1crypto/algos.py", line 24, in <module> from ._int import fill_width File "/usr/local/lib/python3.6/dist-packages/asn1crypto/_int.py", line 56, in <module> from ._perf._big_num_ctypes import libcrypto File "/usr/local/lib/python3.6/dist-packages/asn1crypto/_perf/_big_num_ctypes.py", line 35, in <module> libcrypto_path = find_library(b'crypto' if sys.version_info < (3,) else 'crypto') File "/usr/lib/python3.6/ctypes/util.py", line 313, in find_library return _findSoname_ldconfig(name) or \\ File "/usr/lib/python3.6/ctypes/util.py", line 282, in _findSoname_ldconfig env={'LC_ALL': 'C', 'LANG': 'C'}) as p: File "/usr/local/lib/python3.6/dist-packages/gevent/subprocess.py", line 619, in __init__ restore_signals, start_new_session) File "/usr/local/lib/python3.6/dist-packages/gevent/subprocess.py", line 1474, in _execute_child data = errpipe_read.read() File "/usr/local/lib/python3.6/dist-packages/gevent/_fileobjectposix.py", line 113, in readall data = self.__read(DEFAULT_BUFFER_SIZE) File "/usr/local/lib/python3.6/dist-packages/gevent/_fileobjectposix.py", line 108, in __read self.hub.wait(self._read_event) File "src/gevent/_hub_primitives.py", line 46, in gevent.__hub_primitives.WaitOperationsGreenlet.wait File "src/gevent/_hub_primitives.py", line 55, in gevent.__hub_primitives.WaitOperationsGreenlet.wait File "src/gevent/_waiter.py", line 151, in gevent.__waiter.Waiter.get File "src/gevent/_greenlet_primitives.py", line 59, in gevent.__greenlet_primitives.SwitchOutGreenletWithLoop.switch File "src/gevent/_greenlet_primitives.py", line 59, in gevent.__greenlet_primitives.SwitchOutGreenletWithLoop.switch File "src/gevent/_greenlet_primitives.py", line 63, in gevent.__greenlet_primitives.SwitchOutGreenletWithLoop.switch File "src/gevent/__greenlet_primitives.pxd", line 35, in gevent.__greenlet_primitives._greenlet_switch gevent.exceptions.LoopExit: This operation would block forever \tHub: <Hub '' at 0x7f929cbe9b70 epoll default pending=0 ref=0 fileno=18 thread_ident=0x7f92b8dd4780> \tHandles: []
tried all permutation of import order and also tried monkey patching still getting the error.
The code is working on development server with same python,flask and paramiko versions.
Probably it has something to do with wsgi module?