Flask with paramiko based ssh client gives gevent LoopExit exception - hbknjr - Dec-24-2018
I am getting gevent LoopExit error when using paramiko based ssh client with flask server.
Sample code to reproduce the problem.
#!usr/bin/python3.6
#from gevent import monkey; monkey.patch_all()
from pssh.clients.native import SSHClient
import flask
from flask import request
app = flask.Flask(__name__)
app.config["DEBUG"] = False
def checkcpu(ip):
cmd_cpu_usage = '''awk '{u=$2+$4; t=$2+$4+$5; if (NR==1){u1=u; t1=t;} else print ($2+$4-u1) * 100 / (t-t1) "%"; }' <(grep 'cpu ' /proc/stat) <(sleep 1;grep 'cpu ' /proc/stat)'''
server = SSHClient(ip,user='user',password='password')
output = server.run_command(cmd_cpu_usage)
status = ''
for line in output[2]:
status += line
return status
@app.route('/', methods=['GET'])
def home():
return '<h1>_WORKS_<h1>'
@app.route('/v1/check/cpu', methods=['GET'])
def api_cpu_check():
if 'ip' in request.args:
ip = request.args['ip']
else:
return "Error: IP required"
return checkcpu(ip) Error: ERROR in app: Exception on /v1/check/cpu [GET]
Traceback (most recent call last):
File "/usr/local/lib/python3.6/dist-packages/flask/app.py", line 2292, in wsgi_app
response = self.full_dispatch_request()
File "/usr/local/lib/python3.6/dist-packages/flask/app.py", line 1815, in full_dispatch_request
rv = self.handle_user_exception(e)
File "/usr/local/lib/python3.6/dist-packages/flask/app.py", line 1718, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "/usr/local/lib/python3.6/dist-packages/flask/_compat.py", line 35, in reraise
raise value
File "/usr/local/lib/python3.6/dist-packages/flask/app.py", line 1813, in full_dispatch_request
rv = self.dispatch_request()
File "/usr/local/lib/python3.6/dist-packages/flask/app.py", line 1799, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "/var/www/az/test.py", line 33, in api_cpu_check
return checkcpu(ip)
File "/var/www/az/test.py", line 16, in checkcpu
server = SSHClient(ip,user='user',password='*********')
File "/usr/local/lib/python3.6/dist-packages/pssh/clients/native/single.py", line 130, in __init__
self._connect(self._host, self.port)
File "/usr/local/lib/python3.6/dist-packages/pssh/clients/native/single.py", line 209, in _connect
self.sock.connect((host, port))
File "/usr/local/lib/python3.6/dist-packages/gevent/_socket3.py", line 329, in connect
address = _socketcommon._resolve_addr(self._sock, address)
File "/usr/local/lib/python3.6/dist-packages/gevent/_socketcommon.py", line 376, in _resolve_addr
r = getaddrinfo(host, None, sock.family)
File "/usr/local/lib/python3.6/dist-packages/gevent/_socketcommon.py", line 208, in getaddrinfo
return get_hub().resolver.getaddrinfo(host, port, family, type, proto, flags)
File "/usr/local/lib/python3.6/dist-packages/gevent/resolver/thread.py", line 65, in getaddrinfo
return self.pool.apply(_socket.getaddrinfo, args, kwargs)
File "/usr/local/lib/python3.6/dist-packages/gevent/pool.py", line 159, in apply
return self.spawn(func, *args, **kwds).get()
File "src/gevent/event.py", line 381, in gevent._event.AsyncResult.get
File "src/gevent/event.py", line 406, in gevent._event.AsyncResult.get
File "src/gevent/event.py", line 117, in gevent._event._AbstractLinkable._wait_core
File "src/gevent/event.py", line 119, in gevent._event._AbstractLinkable._wait_core
File "src/gevent/_greenlet_primitives.py", line 59, in gevent.__greenlet_primitives.SwitchOutGreenletWithLoop.switch
File "src/gevent/_greenlet_primitives.py", line 59, in gevent.__greenlet_primitives.SwitchOutGreenletWithLoop.switch
File "src/gevent/_greenlet_primitives.py", line 63, in gevent.__greenlet_primitives.SwitchOutGreenletWithLoop.switch
File "src/gevent/__greenlet_primitives.pxd", line 35, in gevent.__greenlet_primitives._greenlet_switch
gevent.exceptions.LoopExit: This operation would block forever
\tHub: <Hub '' at 0x7f929d9a3828 epoll default pending=0 ref=0 fileno=18 resolver=<gevent.resolver_thread.Resolver at 0x7f9298eeacf8 pool=<ThreadPool at 0x7f929b9ae1d0 0/1/10 hub=<Hub at 0x7f929d9a3828 thread_ident=0x140268143462272>>> threadpool=<ThreadPool at 0x7f929b9ae1d0 0/1/10 hub=<Hub at 0x7f929d9a3828 thread_ident=0x140268143462272>> thread_ident=0x7f92b8dd4780>
\tHandles:
[]
This code works on development server on windows...But gives the above error with when run on linux server as wsgi application.
This didn't work for me, getting exact same error. Even tried to do monkeypatching in wsgi file like:
#!/usr/bin/python3.6
from gevent import monkey; monkey.patch_all()
import sys
sys.path.insert(0,"/var/www/testapp/")
from test import app as application I am not familiar with gevent.
Any help will be appreciated.
RE: Flask with paramiko based ssh client gives gevent LoopExit exception - Gribouillis - Dec-24-2018
The error message says that it fails on the line server = SSHClient(ip,user='user',password='*********') because in the end it may block forever. I don't know this pssh module, but using a timeout= keyword argument in SSHClient() may help solve the issue by preventing the call to block forever.
RE: Flask with paramiko based ssh client gives gevent LoopExit exception - hbknjr - Dec-24-2018
Thanks for replying.
I tried to minimize the waits, making few assumptions with following code:
server = SSHClient(ip,user='user',password='********',timeout=5,num_retries=2,retry_delay=1,keepalive_seconds=0)
#timeout : ssh session timeout
#num_retries: connection attempts
#keepalive_seconds: time interval for keep alive packet to be sent. I have no idea about gevent but this should put a limit on blocks/waits
But same error persists.
RE: Flask with paramiko based ssh client gives gevent LoopExit exception - hbknjr - Dec-25-2018
Changed code to directly use paramiko.
#!usr/bin/python3.6
#from gevent import monkey; monkey.patch_all()
import flask
from flask import request
import paramiko
app = flask.Flask(__name__)
app.config["DEBUG"] = False
def exucute_cmd(cmd,ip,user,pw,port=22):
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(ip, port, user, pw)
shell = client.invoke_shell()
_, stdout, stderr = client.exec_command(cmd)
out=''
for line in stdout.readlines():
out+=line
if not out:
for line in stderr.readlines():
out+=line
shell.close()
client.close()
return out
def checkcpu(ip):
cmd_cpu_usage = '''awk '{u=$2+$4; t=$2+$4+$5; if (NR==1){u1=u; t1=t;} else print ($2+$4-u1) * 100 / (t-t1) "%"; }' <(grep 'cpu ' /proc/stat) <(sleep 1;grep 'cpu ' /proc/stat)'''
return exucute_cmd(cmd_cpu_usage,ip,'user','*******') Getting Error on Paramiko Import:
Error: Traceback (most recent call last):
File "src/gevent/_waiter.py", line 116, in gevent.__waiter.Waiter.switch
AssertionError: Can only use Waiter.switch method from the Hub greenlet
2018-12-25T07:30:17Z <io at 0x7f929b9716c8 fd=24 events=READ active callback=<bound method Waiter.switch of <gevent.__waiter.Waiter object at 0x7f929b98df98>> args=(<gevent.__waiter.Waiter object at 0x7f929b98df98>,)> failed with AssertionError
mod_wsgi (pid=6149): Failed to exec Python script file '/var/www/az/az.wsgi'.
mod_wsgi (pid=6149): Exception occurred processing WSGI script '/var/www/az/az.wsgi'.
Traceback (most recent call last):
File "/var/www/az/az.wsgi", line 8, in <module>
from test import app as application
File "/var/www/az/test.py", line 4, in <module>
import paramiko
File "/usr/local/lib/python3.6/dist-packages/paramiko/__init__.py", line 22, in <module>
from paramiko.transport import SecurityOptions, Transport
File "/usr/local/lib/python3.6/dist-packages/paramiko/transport.py", line 89, in <module>
from paramiko.dsskey import DSSKey
File "/usr/local/lib/python3.6/dist-packages/paramiko/dsskey.py", line 27, in <module>
from cryptography.hazmat.primitives.asymmetric.utils import (
File "/usr/local/lib/python3.6/dist-packages/cryptography/hazmat/primitives/asymmetric/utils.py", line 9, in <module>
from asn1crypto.algos import DSASignature
File "/usr/local/lib/python3.6/dist-packages/asn1crypto/algos.py", line 24, in <module>
from ._int import fill_width
File "/usr/local/lib/python3.6/dist-packages/asn1crypto/_int.py", line 56, in <module>
from ._perf._big_num_ctypes import libcrypto
File "/usr/local/lib/python3.6/dist-packages/asn1crypto/_perf/_big_num_ctypes.py", line 35, in <module>
libcrypto_path = find_library(b'crypto' if sys.version_info < (3,) else 'crypto')
File "/usr/lib/python3.6/ctypes/util.py", line 313, in find_library
return _findSoname_ldconfig(name) or \\
File "/usr/lib/python3.6/ctypes/util.py", line 282, in _findSoname_ldconfig
env={'LC_ALL': 'C', 'LANG': 'C'}) as p:
File "/usr/local/lib/python3.6/dist-packages/gevent/subprocess.py", line 619, in __init__
restore_signals, start_new_session)
File "/usr/local/lib/python3.6/dist-packages/gevent/subprocess.py", line 1474, in _execute_child
data = errpipe_read.read()
File "/usr/local/lib/python3.6/dist-packages/gevent/_fileobjectposix.py", line 113, in readall
data = self.__read(DEFAULT_BUFFER_SIZE)
File "/usr/local/lib/python3.6/dist-packages/gevent/_fileobjectposix.py", line 108, in __read
self.hub.wait(self._read_event)
File "src/gevent/_hub_primitives.py", line 46, in gevent.__hub_primitives.WaitOperationsGreenlet.wait
File "src/gevent/_hub_primitives.py", line 55, in gevent.__hub_primitives.WaitOperationsGreenlet.wait
File "src/gevent/_waiter.py", line 151, in gevent.__waiter.Waiter.get
File "src/gevent/_greenlet_primitives.py", line 59, in gevent.__greenlet_primitives.SwitchOutGreenletWithLoop.switch
File "src/gevent/_greenlet_primitives.py", line 59, in gevent.__greenlet_primitives.SwitchOutGreenletWithLoop.switch
File "src/gevent/_greenlet_primitives.py", line 63, in gevent.__greenlet_primitives.SwitchOutGreenletWithLoop.switch
File "src/gevent/__greenlet_primitives.pxd", line 35, in gevent.__greenlet_primitives._greenlet_switch
gevent.exceptions.LoopExit: This operation would block forever
\tHub: <Hub '' at 0x7f929cbe9b70 epoll default pending=0 ref=0 fileno=18 thread_ident=0x7f92b8dd4780>
\tHandles:
[]
tried all permutation of import order and also tried monkey patching still getting the error.
The code is working on development server with same python,flask and paramiko versions.
Probably it has something to do with wsgi module?
|