![]() |
Why wont subprocess call work? - Printable Version +- Python Forum (https://python-forum.io) +-- Forum: Python Coding (https://python-forum.io/forum-7.html) +--- Forum: General Coding Help (https://python-forum.io/forum-8.html) +--- Thread: Why wont subprocess call work? (/thread-26321.html) |
Why wont subprocess call work? - steve_shambles - Apr-28-2020 I am interested in getting this snippet to work but it uses call from subprocess module which is causing errors Quote:Found 8 gists in 1 pages I have replaced the first call that just created a directory with the os equivalent to get past one call error only to be met by two more calls near end of the code. I tried to replace those with popen but it didn't work. Does anyone have an idea on what is going on here please? """scrape a github users gists. origin: https://gist.github.com/dreizehnutters/6ac65948b506fbcd9f67ad0dd11ce50a """ from json import load import os import subprocess from subprocess import call from sys import argv from urllib import request GITHUB = "https://api.github.com/users/" USER = "steveshambles" if not os.path.isdir(USER): os.mkdir(USER) COUNT = 0 PERPAGE = 100 GISTCOUNT = load(request.urlopen(f"{GITHUB}{USER}"))['public_gists'] PAGES = (GISTCOUNT//PERPAGE)+2 print(f"Found {GISTCOUNT} gists in {PAGES-1} pages") for page in range(1, PAGES): for gist in load(request.urlopen(f"{GITHUB}{USER}/gists?page={page}&per_page={PERPAGE}")): COUNT += 1 gistd = list(gist['files'].keys())[0] gistUrl = f"git://gist.github.com/{gist['id']}.git" path = f"{USER}/{gist['id']}" print(f"[*] cloning {gistUrl} from page {page} [{COUNT}/{GISTCOUNT}]") #subprocess.Popen(['git', 'clone', '--quiet', gistUrl, path]) call(['git', 'clone', '--quiet', gistUrl, path]) call(['mv', path, f"{USER}/{gistd}"])>>> RE: Why wont subprocess call work? - DeaD_EyE - Apr-28-2020 It seems that git is not in your path. You can use the absolute path or you have to change your PATH environment variable. The code could be improved if you use function and bring everything in the right order. You can try this: PS C:\Users\Admin\Desktop\gists> py -3 Python 3.8.2 (tags/v3.8.2:7b3ab59, Feb 25 2020, 23:03:10) [MSC v.1916 64 bit (AMD64)] on win32 Type "help", "copyright", "credits" or "license" for more information. >>> import shutil >>> shutil.which("git") 'C:\\Program Files\\Git\\cmd\\git.EXE' >>>If you get no result, then git.exe is not in the Path. Here an improved version. But you still need to solve the missing git in path. """ scrape a github users gists. origin: https://gist.github.com/dreizehnutters/6ac65948b506fbcd9f67ad0dd11ce50a """ import argparse import sys import json from pathlib import Path from subprocess import call from urllib.request import urlopen from urllib.error import HTTPError, URLError GITHUB = "https://api.github.com/users/" PERPAGE = 100 def get_gist_count(user): gistcount = json.load(urlopen(f"{GITHUB}{user}"))['public_gists'] pages = gistcount // PERPAGE + 2 return gistcount, pages def get_gist_page(user, page): rep = urlopen(f"{GITHUB}{user}/gists?page={page}&per_page={PERPAGE}") return json.load(rep) def get_gists(user, gistcount, pages): count = 0 for page in range(1, pages): for gist in get_gist_page(user, page): count += 1 gist_d = list(gist['files'].keys())[0] gist_id = gist['id'] gist_url = f"git://gist.github.com/{gist_id}.git" src_path = user / gist_id dst_path = user / gist_d print(f"[*] cloning {gist_url} from page {page} [{count}/{gistcount}]") src_path.unlink(missing_ok=True) call(['git', 'clone', '--quiet', gist_url, src_path]) src_path.replace(dst_path) if __name__ == "__main__": parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("user", type=Path, help="Github user account") args = parser.parse_args() try: gistcount, pages = get_gist_count(args.user) except (HTTPError, URLError) as e: print(e, file=sys.stderr) sys.exit(1) args.user.mkdir(exist_ok=True) print(f"Found {gistcount} gists in {pages-1} pages") try: get_gists(args.user, gistcount, pages) except (HTTPError, URLError) as e: print(e, file=sys.stderr) sys.exit(1)If you're on Windows, then check the PermissionError. I get this always where src_path.unlink() should delete it. On linux you should not have this strange issues. RE: Why wont subprocess call work? - ndc85430 - Apr-28-2020 Why are you using subprocess anyway, instead of using e.g. pygit2?
RE: Why wont subprocess call work? - steve_shambles - Apr-28-2020 I didn't write it. origin: https://gist.github.com/dreizehnutters/6ac65948b506fbcd9f67ad0dd11ce50a (Apr-28-2020, 01:48 PM)ndc85430 Wrote: Why are you using Yes I'm on Windows 7 using Python 3.67 still won't work. I'm going to look at the old way of scraping rather than rely on github to help. thanks for your help. (Apr-28-2020, 12:53 PM)DeaD_EyE Wrote: If you're on Windows, then check the PermissionError. |