Jun-07-2018, 06:43 PM
Lars60+ crafted the first version of this. When I run his version, it runs fine. I don't know what I did to mess mine up. I didn't change much but I've learned it doesn't take much to turn it right on its head! Let me know if you want to see his to compare. I am downloading different reports but from the same website.
As always - any help is most appreciated!
As always - any help is most appreciated!
import requests from bs4 import BeautifulSoup from pathlib import Path import CheckInternet import sys class GetCompletions: def __init__(self, infile): self.check_network = CheckInternet.CheckInternet() self.homepath = Path('.') self.rootpath = self.homepath / '..' self.datapath = self.rootpath / 'data' self.commandpath = self.datapath / 'command_files' self.wellgeopath = self.datapath / 'wellgeo' self.htmlpath = self.datapath / 'html' self.reportspath = self.datapath / 'reports' if self.check_network.check_availability(): # use: Api_May_27_2018.txt for testing # self.infilename = 'Api_May_27_2018.txt' self.infilename = input('Please enter api filename: ') self.infile = self.commandpath / self.infilename self.api = [] with self.infile.open() as f: for line in f: self.api.append(line.strip()) self.fields = ['API Number', 'Field', 'Formation', 'Well', 'location'] self.get_all_pages() self.parse_and_save(getpdfs=True) else: print('Internet access required, and not found.') print('Please make Internet available and try again') def get_url(self): for entry in self.api: print("http://http://wogcc.state.wy.us/coreapi.cfm?API={}".format(entry[3:10])) yield (entry, "http://http://wogcc.state.wy.us/coreapi.cfm?API={}".format(entry[3:10])) def get_all_pages(self): for entry, url in self.get_url(): print('Fetching main page for entry: {}'.format(entry)) response = requests.get(url) if response.status_code == 200: filename = self.htmlpath / 'api_{}.html'.format(entry) with filename.open('w') as f: f.write(response.text) else: print('error downloading {}'.format(entry)) def parse_and_save(self, getpdfs=False): filelist = [file for file in self.htmlpath.iterdir() if file.is_file()] for file in filelist: with file.open('r') as f: soup = BeautifulSoup(f.read(), 'lxml') if getpdfs: links = soup.find_all('a') for link in links: url = link['href'] if 'www' in url: continue print('downloading pdf at: {}'.format(url)) p = url.index('=') response = requests.get(url, stream=True, allow_redirects=False) if response.status_code == 200: try: header_info = response.headers['Content-Disposition'] idx = header_info.index('filename') filename = self.wellgeopath / header_info[idx + 9:] except ValueError: filename = self.wellgeopath / 'comp{}.pdf'.format(url[p + 1:]) print("couldn't locate filename for {} will use: {}".format(file, filename)) except KeyError: filename = self.wellgeopath / 'comp{}.pdf'.format(url[p + 1:]) print('got KeyError on {}, response.headers = {}'.format(file, response.headers)) print('will use name: {}'.format(filename)) print(response.headers) with filename.open('wb') as f: f.write(response.content) sfname = self.reportspath / 'summary_{}.txt'.format((file.name.split('_'))[1].split('.')[0][3:10]) tds = soup.find_all('td') with sfname.open('w') as f: for td in tds: if td.text: if any(field in td.text for field in self.fields): f.write('{}\n'.format(td.text)) # Delete html file when finished file.unlink() if __name__ == '__main__': GetCompletions('apis.txt')
Error:C:\Python365\python.exe "O:/Python/WellInfo - GEO/src/FetchCompletions.py"
Please enter api filename: Api_May_27_2018.txt
http://http://wogcc.state.wy.us/coreapi.cfm?API=0527389
Fetching main page for entry: 49005273890000
Traceback (most recent call last):
File "C:\Python365\lib\site-packages\urllib3\connection.py", line 141, in _new_conn
(self.host, self.port), self.timeout, **extra_kw)
File "C:\Python365\lib\site-packages\urllib3\util\connection.py", line 60, in create_connection
for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):
File "C:\Python365\lib\socket.py", line 745, in getaddrinfo
for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
socket.gaierror: [Errno 11004] getaddrinfo failed
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Python365\lib\site-packages\urllib3\connectionpool.py", line 601, in urlopen
chunked=chunked)
File "C:\Python365\lib\site-packages\urllib3\connectionpool.py", line 357, in _make_request
conn.request(method, url, **httplib_request_kw)
File "C:\Python365\lib\http\client.py", line 1239, in request
self._send_request(method, url, body, headers, encode_chunked)
File "C:\Python365\lib\http\client.py", line 1285, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "C:\Python365\lib\http\client.py", line 1234, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "C:\Python365\lib\http\client.py", line 1026, in _send_output
self.send(msg)
File "C:\Python365\lib\http\client.py", line 964, in send
self.connect()
File "C:\Python365\lib\site-packages\urllib3\connection.py", line 166, in connect
conn = self._new_conn()
File "C:\Python365\lib\site-packages\urllib3\connection.py", line 150, in _new_conn
self, "Failed to establish a new connection: %s" % e)
urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPConnection object at 0x000000000372ECC0>: Failed to establish a new connection: [Errno 11004] getaddrinfo failed
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Python365\lib\site-packages\requests\adapters.py", line 440, in send
timeout=timeout
File "C:\Python365\lib\site-packages\urllib3\connectionpool.py", line 639, in urlopen
_stacktrace=sys.exc_info()[2])
File "C:\Python365\lib\site-packages\urllib3\util\retry.py", line 388, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='http', port=80): Max retries exceeded with url: //wogcc.state.wy.us/coreapi.cfm?API=0527389 (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x000000000372ECC0>: Failed to establish a new connection: [Errno 11004] getaddrinfo failed',))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "O:/Python/WellInfo - GEO/src/FetchCompletions.py", line 95, in <module>
GetCompletions('apis.txt')
File "O:/Python/WellInfo - GEO/src/FetchCompletions.py", line 32, in __init__
self.get_all_pages()
File "O:/Python/WellInfo - GEO/src/FetchCompletions.py", line 46, in get_all_pages
response = requests.get(url)
File "C:\Python365\lib\site-packages\requests\api.py", line 72, in get
return request('get', url, params=params, **kwargs)
File "C:\Python365\lib\site-packages\requests\api.py", line 58, in request
return session.request(method=method, url=url, **kwargs)
File "C:\Python365\lib\site-packages\requests\sessions.py", line 508, in request
resp = self.send(prep, **send_kwargs)
File "C:\Python365\lib\site-packages\requests\sessions.py", line 618, in send
r = adapter.send(request, **kwargs)
File "C:\Python365\lib\site-packages\requests\adapters.py", line 508, in send
raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: HTTPConnectionPool(host='http', port=80): Max retries exceeded with url: //wogcc.state.wy.us/coreapi.cfm?API=0527389 (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x000000000372ECC0>: Failed to establish a new connection: [Errno 11004] getaddrinfo failed',))
Process finished with exit code 1