Python Forum
Thread Rating:
  • 0 Vote(s) - 0 Average
  • 1
  • 2
  • 3
  • 4
  • 5
Python decorators.
#1
Hi, I am just studying python programming. I have a task to create Image classifyier and I have decided to make it as autonomic module. I have tryied to get around google images search for data loading. On this point I have met this "problem": I have needed an decorator function which, I thought, would be able to use class's instance methods (__urls_loading and __images_loading private methods). I have used this construction (in the picture, it works) and my question is "Is there any easier way to do the same, if not, is there any chance it will be added in next python versions?" (I want to say sorry about docs, code has not been finished yet, also where is another exeption, which is not connected with the thing I ask)

import fastai.vision as fv
import os
from selenium import webdriver
from clicker import element_clicker
from selenium.webdriver.common.keys import Keys
import selenium
import ssl
import requests
import urllib3
import shutil
from selenium import common
import sys
import constants as const
import time


urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)


class ImageClassifier:
    """ Image classifier class. """

    def __init__(self, classifying_object, garbage_dir_name="garbage",
                 path_to_webdriver="webdriver/chromedriver.exe", data_dir_name="data"):
        """
        Image classifier instance initialization.
        :param classifying_object:
        :type classifying_object: str
        :param garbage_dir_name:
        :type garbage_dir_name: str
        :param data_dir_name:
        :type data_dir_name: str
        :param path_to_webdriver:
        :type path_to_webdriver: str
        :return Image classifier instance.
        """
        # Setting instance initial attributes.
        self.__classifying_object = classifying_object
        self.__data_dir_name = data_dir_name
        self.__new_dir_creation(f"{data_dir_name}")
        self.__objects_dir_name = classifying_object
        self.__garbage_dir_name = garbage_dir_name
        self.__path_to_webdriver = path_to_webdriver
        self.__webdriver = None
        self.__savings_counter = 0

        # Creating initial data directories.
        self.__new_dir_creation(f"{data_dir_name}/images/{classifying_object}")
        self.__new_dir_creation(f"{data_dir_name}/images/{garbage_dir_name}")

        # Garbage_downloading.
        self.__garbage_downloading()

    @staticmethod
    def webdriver_deco(instance):
        def deco(func):
            def wrapper(*args, **kwargs):
                try:
                    instance.__set_webdriver()
                    func(*args, **kwargs)
                    instance.__webdriver.close()
                except (ssl.SSLEOFError,
                        urllib3.exceptions.MaxRetryError,
                        requests.exceptions.SSLError):
                    instance.__set_webdriver()
                    func(*args, **kwargs)
            return wrapper
        return deco

    def __set_webdriver(self):
        # Driver options setup.
        options = webdriver.ChromeOptions()

        # Hidden browser argument.
        # options.add_argument('headless')

        # Setting downloading path in chrome understandable format.
        preferences = {
            "download.default_directory": f'{sys.path[0]}'.replace('/', '\\')
        }

        options.add_experimental_option("prefs", preferences)

        creation_params = {
            "executable_path": self.__path_to_webdriver,
            "chrome_options": options
        }

        self.__webdriver = webdriver.Chrome(**creation_params)

    @staticmethod
    def __new_dir_creation(directory, abs_path=False):
        """
        New directory creation static method.
        :param directory:
        :param abs_path:
        :type directory: str
        :type abs_path: bool
        :return: None
        """
        directory = f"{os.getcwd()}/{directory}" if not abs_path else directory

        if not os.path.exists(f"{directory}"):
            print(directory)
            os.makedirs(f"{directory}")

    def __urls_loading(self, classifying_object):
        """
        Urls loading method.
        :return: None
        """
        @ImageClassifier.webdriver_deco(self)
        def sub_func(_classifying_object):
            def element_click(path='body', path_type='css',
                              web_driver=self.__webdriver):
                """ Setting driver to clicker function. """
                element_clicker(path=path, path_type=path_type, driver=web_driver)

            if _classifying_object == "garbage":
                _classifying_object = "random images"

            self.__webdriver.get("https://www.google.ru/imghp?hl=ru&tab=wi&ogbl")
            search_field = self.__webdriver.find_element_by_css_selector(const.SEARCH_FIELD_CSS)
            search_field.send_keys(_classifying_object)
            search_field.send_keys(Keys.ENTER)
            js_scripts_path = const.JS_SCRIPTS_PATH

            with open(f"{js_scripts_path}/scrolling_script.js") as js_scrolling_script:
                scroll_script = js_scrolling_script.read()

            self.__webdriver.execute_script(scroll_script)
            time.sleep(5)
            try:
                element_click(path="#smb", path_type="css")
            except common.exceptions.ElementNotInteractableException:
                pass
            except selenium.common.exceptions.NoSuchElementException:
                element_click(path="Показать ещё", path_type="text")

            with open(f"{js_scripts_path}/image_urls_catching.txt") as js_image_urls_catching_script:
                js_script = js_image_urls_catching_script.read()

            image_urls = self.__webdriver.execute_script(js_script)

            if _classifying_object == "random images":
                _classifying_object = "garbage"

            images_urls_len = min(len(image_urls), 50)

            with open(f"{self.__data_dir_name}/urls/{_classifying_object}.csv", "w") as urls_data_file:
                [urls_data_file.write(f"{image_url}\n") for image_url in image_urls[:images_urls_len]]
        sub_func(classifying_object)

    def __images_loading(self, classifying_object):
        """
        Images loading method.
        :return: None
        """
        @ImageClassifier.webdriver_deco(self)
        def sub_func(_classifying_object):
            self.__new_dir_creation(f"{self.__data_dir_name}/images/{_classifying_object}")

            with open(f"{self.__data_dir_name}/urls/{_classifying_object}.csv") as urls_data_file:
                urls_data = [*map((lambda url: url.rstrip()), urls_data_file.readlines())]

            for number, image_url in enumerate(urls_data[self.__savings_counter:]):
                image = requests.get(image_url, stream=True, verify=False)
                image.raw.decode_content = True
                saving_path = f"{self.__data_dir_name}/images/{_classifying_object}/{number}.{image_url[-3:]}"
                with open(saving_path, "wb") as file:
                    shutil.copyfileobj(image.raw, file)
                self.__savings_counter += 1
        sub_func(classifying_object)
        self.__savings_counter = 0

    def __check_images(self, classifying_object):
        """
        Checking images method.
        :return: None
        """
        image_files_path = f"{os.getcwd()}/{self.__data_dir_name}/images/{classifying_object}"
        incorrect_files = []
        for image_file in os.listdir(image_files_path):
            if image_file[-3:] not in ["png", "jpg"]:
                incorrect_files += [image_file]

        for image_file in incorrect_files:
            os.remove(f"{image_files_path}/{image_file}")

        images_path = f"{self.__data_dir_name}/images/{classifying_object}"
        fv.verify_images(images_path, delete=True, max_size=200)

    def data_loading(self, classifying_object=None):
        """
        Data loading method.
        :return: None
        """
        classifying_object = self.__classifying_object if not classifying_object else classifying_object
        self.__urls_loading(classifying_object)
        self.__images_loading(classifying_object)
        self.__check_images(classifying_object)

    def __garbage_downloading(self):
        """
        Garbage loading function.
        :return: None
        """
        current_classifying_object = self.__classifying_object
        self.__classifying_object = "garbage"
        self.data_loading()
        self.__classifying_object = current_classifying_object

    def __reload_data(self):
        """
        Updating data method.
        :return: None
        """
        self.__data_loading()

    def classifier_object_change(self, new_classifying_object, reload=False):
        """
        Classifying object changing method.
        :param new_classifying_object:
        :param reload:
        :type new_classifying_object: str
        :type reload: bool
        :return: None
        """
        self.__classifying_object = new_classifying_object
        if reload:
            self.__reload_data()


classifier = ImageClassifier("python")
classifier.data_loading()
Reply


Possibly Related Threads…
Thread Author Replies Views Last Post
  Decorators @ annotation drcl 3 349 Feb-24-2024, 06:12 AM
Last Post: Gribouillis
  Does @ at sign used for tother than decorators? ggpython000 1 497 Oct-12-2023, 09:08 AM
Last Post: buran
  Variable Scopes Decorators oclmedyb 6 2,670 Jan-15-2021, 02:13 PM
Last Post: oclmedyb
  Decorators and return statements JonEdward 2 1,857 Jul-24-2020, 05:02 PM
Last Post: JonEdward
  How to print cache from Decorators with Memoization OlgaM 2 2,017 Jan-29-2020, 05:06 PM
Last Post: OlgaM
  Decorators yksingh1097 2 2,474 Aug-14-2018, 01:44 PM
Last Post: yksingh1097
  learning decorators Prince_Bhatia 6 3,240 Aug-13-2018, 02:28 PM
Last Post: Prince_Bhatia
  decorators within decorators mp3909 6 4,196 Apr-02-2018, 09:47 AM
Last Post: hf8bm
  decorators mp3909 9 5,489 Mar-25-2018, 05:28 PM
Last Post: mp3909

Forum Jump:

User Panel Messages

Announcements
Announcement #1 8/1/2020
Announcement #2 8/2/2020
Announcement #3 8/6/2020