Python Forum

Full Version: Corpora catalog for NLTK in json format
You're currently viewing a stripped down version of our content. View the full version with proper formatting.
Here's code to create a JSON file with an index of NLTK Corpora data.
The json file is in the form of a dictionary with package id as the key, followed by the following fields (values) as a large tuple:
pkid, pkname, pkauthor, pklanguages, pkurl, pkchecksum, pksize, pksubdir, pkunzip, pkunzipped_size, pkwebpage,
pkcontact, pklicense, pkcopyright, pknote, pksample

"""
The MIT License (MIT)

Copyright (c) <2016> <Larz60+ aka: Larz60p>

Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

"""
import os
from bs4 import BeautifulSoup
import HasInternet as hsint
import GetUrl as gurl
import json

class CorporaData:
    def __init__(self, url='http://www.nltk.org/nltk_data/'):
        self.Corpora = {}
        corpus_url = url
        datafname = 'CorpusSource.html'
        geturl = gurl.GetUrl()
        self.has_internet = hsint.has_connection()
        if self.has_internet:
            self.data = geturl.get_url(corpus_url, ret=True, tofile=datafname)
        else:
            if os.path.isfile(datafname):
                with open(datafname, 'r') as f:
                    self.data = f.read()

        soup = BeautifulSoup(self.data, "html.parser")
        packages = soup.find_all('package')
        print(packages)
        for package in packages:
            pkauthor = package.get('author')
            print('Author: {}'.format(pkauthor))

            pkchecksum = package.get('checksum')
            print('Checksum: {}'.format(pkchecksum))

            pkid = package.get('id')
            print('Id: {}'.format(pkid))

            pklanguages = package.get('languages')
            print('Languages: {}'.format(pklanguages))

            pkname = package.get('name')
            print('Name: {}'.format(pkname))

            pksize = package.get('size')
            print('Size: {}'.format(pksize))

            pksubdir = package.get('subdir')
            print('Subdir: {}'.format(pksubdir))

            pkunzip = package.get('unzip')
            print('Unzip: {}'.format(pkunzip))

            pkunzipped_size = package.get('unzipped_size')
            print('Unzipped_size: {}'.format(pkunzipped_size))

            pkurl = package.get('url')
            print('url: {}'.format(pkurl))

            pkwebpage = package.get('webpage')
            print('Webpage: {}'.format(pkwebpage))

            pkcontact = package.get('contact')
            print('Contact: {}'.format(pkcontact))

            pklicense = package.get('license')
            print('License: {}'.format(pklicense))

            pkcopyright = package.get('copyright')
            print('Copyright: {}'.format(pkcopyright))

            pknote = package.get('note')
            print('Note: {}'.format(pknote))

            pksample = package.get('sample')
            print('Sample: {}'.format(pksample))
            print()
            key = repr(pkid)
            print('Key: {}'.format(key))
            self.Corpora[key] = (pkid, pkname, pkauthor, pklanguages, pkurl, pkchecksum,
                                 pksize, pksubdir, pkunzip, pkunzipped_size, pkwebpage,
                                 pkcontact, pklicense, pkcopyright, pknote, pksample)

            with open("Corpora.json", "w") as f:
                j = json.dumps(self.Corpora)
                f.write(j)


if __name__ == '__main__':
    CorporaData()
Sample of printed output:
Output:
key: 'maxent_ne_chunker' Author: Australian Broadcasting Commission Checksum: ffb36b67ff24cbf7daaf171c897eb904 Id: abc Languages: None Name: Australian Broadcasting Commission 2006 Size: 1487851 Subdir: corpora Unzip: 1 Unzipped_size: 4054966 url: https://raw.githubusercontent.com/nltk/nltk_data/gh-pages/packages/corpora/abc.zip Webpage: http://www.abc.net.au/ Contact: None License: None Copyright: None Note: None Sample: None Key: 'abc' Author: None Checksum: ae529a1c5f13d6074f5b0d68d8edb537 Id: alpino Languages: None Name: Alpino Dutch Treebank Size: 2797255 Subdir: corpora Unzip: 1 Unzipped_size: 21604821 url: https://raw.githubusercontent.com/nltk/nltk_data/gh-pages/packages/corpora/alpino.zip Webpage: http://www.let.rug.nl/~vannoord/trees/ Contact: Gertjan van Noord License: Distributed with permission of Gertjan van Noord Copyright: None Note: None Sample: None
Larz60+

Need the GetUrl module for above to work:
import urllib.request as ur
import os
from time import sleep
import sys


class GetUrl:
    def __init__(self, returndata=False):
        self.returndata = returndata

    def get_url(self, url, ret=False, tofile=None, bin=False):
        rdata = None
        head, tail = os.path.split(url)
        try:
            if tofile:
                if os.path.exists(tofile):
                    os.remove(tofile)
                if bin:
                    with open(tofile, 'wb') as f:
                        rdata = ur.urlopen(url).read()
                        f.write(rdata)
                else:
                    with open(tofile, 'w') as f:
                        rdata = ur.urlopen(url).read().decode('utf8')
                        f.write(rdata)
                sleep(.5)
            if ret:
                return rdata
        except:
            print("Unexpected error:", sys.exc_info()[0])

if __name__ == '__main__':
    url = 'ftp://ftp.nasdaqtrader.com/symboldirectory/phlxListedStrikesWithOptionIds.zip'
    tofile = 'phlxListedStrikesWithOptionIds.zip'
    p = GetUrl()
    p.get_url(url, tofile, bin=True)
Sooner or later I'll get it all.
You also need HasInternet.py
import subprocess


def has_connection():
    website = "google.com"
    ping = subprocess.Popen(["ping", website], stdout=subprocess.PIPE,
                            stderr=subprocess.PIPE)
    out, error = ping.communicate()
    if 'Sent = 4, Received = 4' in out.decode('utf-8'):
        return True
    return False

if __name__ == '__main__':
    print(has_connection())
We went over this one in a previous thread, and this is the final (until not final anymore) version that we all agreed on
I have modified this code quite a bit, and have put it on GitHub at:
https://github.com/Larz60p/NLTK-Corpora-Catalog

The output is a better JSON file (dictionary format)
A sample of the printed output is:

Output:
RecId: RecId61     url: https://raw.githubusercontent.com/nltk/nltk_data/gh-pages/packages/corpora/sentiwordnet.zip     unzip: 1     license: Creative Commons Attribution ShareAlike 3.0 Unported license     id: sentiwordnet     webpage: http://sentiwordnet.isti.cnr.it/     subdir: corpora     author: Stefano Baccianella, Andrea Esuli, and Fabrizio Sebastiani     copyright: Copyright (C) 2013 SentiWordNet Project     checksum: 5043f00829b7db4dd5f21507e092b76a     name: SentiWordNet     size: 4686546     unzipped_size: 13591402