Hi,
I've been working on a file counter that will cout files from a folder in linux (CentOS7) that can output to to a page which i can use prometheus to collect the data but i'm not having much luck getting it to ready what i need can anyone help with what could be wrong?
count_exporter.py
locations for files are:
I have got the script to run and outputs data onto the httpserver but does not display any metrics for the file count.
example output (localhost:9666/metrics)
Any help would be really helpful! i have also changed the folder/file permissions to 777 on all areas incase that this was the issue.
Thanks.
I've been working on a file counter that will cout files from a folder in linux (CentOS7) that can output to to a page which i can use prometheus to collect the data but i'm not having much luck getting it to ready what i need can anyone help with what could be wrong?
count_exporter.py
#!/usr/bin/env python3 import os import json from argparse import ArgumentParser from prometheus_client import start_http_server, Metric, REGISTRY from prometheus_client.core import GaugeMetricFamily, CounterMetricFamily import time import logging import sys PORT = 9666 # Metrik-Endpoint Port for calling "ip:PORT/metrics" UPDATE_INTERVAL = 10 # in seconds debug = False files = {} states = {} config = "./config.txt" def parse_args(): parser = ArgumentParser() parser.add_argument('-d', '--debug', action='store_true', help='Activate Debug Output on stdout') parser.add_argument('config', help='Path of Config File',nargs='?') args = parser.parse_args() if args.config is None: args.config = "./config.txt" return args def count_files_in_path(path): if debug: print ("Count Files in %s" %path) num_files = len([f for f in os.listdir(path)if os.path.isfile(os.path.join(path, f))]) return num_files def read_config(path): if debug: print ("Open Config %s" %path) file = open(path,"r+") lines = file.readlines() for line in lines: if len(line.split()) < 2: sys.exit("Error in Config File!") path = line.split()[1] name = line.split()[0] files[name] = path states[name] = 0 return def update_matches(): for name, state in states.items(): states[name] = count_files_in_path(files[name]) if debug: print ("Update State for %s: %s" %name,state) return class CustomCollector(object): def collect(self): c = GaugeMetricFamily('file_count', 'Filescount in Path', labels=[ 'name']) for name, state in states.items(): if debug: print ("Collect State for %s: %s" %name,state) c.add_metric(['%s' % name], int(state)) yield c class StreamToLogger(object): """ Fake file-like stream object that redirects writes to a logger instance. """ def __init__(self, logger,terminal, log_level=logging.INFO): self.terminal = terminal self.logger = logger self.log_level = log_level self.linebuf = '' def write(self, buf): self.terminal.write(buf) for line in buf.rstrip().splitlines(): self.logger.log(self.log_level, line.rstrip()) #self.terminal.write(line) logging.basicConfig( level=logging.DEBUG, format='%(asctime)s:%(levelname)s:%(name)s:%(message)s', filename="out.log", filemode='a' ) if __name__ == '__main__': args = parse_args() if args.debug: debug = True config = args.config read_config(config) # Set Logger to Stdout and Stderror stdout_logger = logging.getLogger('STDOUT') sl = StreamToLogger(stdout_logger, sys.stdout, logging.INFO) sys.stdout = sl stderr_logger = logging.getLogger('STDERR') sl = StreamToLogger(stderr_logger, sys.stderr, logging.ERROR) sys.stderr = sl update_matches() print ("Start Listening on Port %s" % PORT) # Start Http Server, waiting for Prometheus Pulls start_http_server(int(PORT)) REGISTRY.register(CustomCollector()) while True: update_matches() time.sleep(UPDATE_INTERVAL)
config.txt [hr] IN /data/IN OUT1 /data/OUT1 OUT2 /data/OUT2 OUT3 /data/OUT3
locations for files are:
Service: /etc/systemd/system/count_exporter.service Script: /etc/count_exporter/count_exporter.py Config: /etc/count_exporter/config.txtCounting loctions:
/data/IN /data/OUT1 /data/OUT2 /data/OUT3I found this on the following link, i've tried to follow this the best i can and had to make some changes to the py script on the site as it was trying to run on 2.7, https://github.com/fpietsch/prometheus-f...t-exporter.
I have got the script to run and outputs data onto the httpserver but does not display any metrics for the file count.
example output (localhost:9666/metrics)
Output:# HELP python_gc_objects_collected_total Objects collected during gc
# TYPE python_gc_objects_collected_total counter
python_gc_objects_collected_total{generation="0"} 371.0
python_gc_objects_collected_total{generation="1"} 7.0
python_gc_objects_collected_total{generation="2"} 0.0
# HELP python_gc_objects_uncollectable_total Uncollectable object found during GC
# TYPE python_gc_objects_uncollectable_total counter
python_gc_objects_uncollectable_total{generation="0"} 0.0
python_gc_objects_uncollectable_total{generation="1"} 0.0
python_gc_objects_uncollectable_total{generation="2"} 0.0
# HELP python_gc_collections_total Number of times this generation was collected
# TYPE python_gc_collections_total counter
python_gc_collections_total{generation="0"} 43.0
python_gc_collections_total{generation="1"} 3.0
python_gc_collections_total{generation="2"} 0.0
# HELP python_info Python platform information
# TYPE python_info gauge
python_info{implementation="CPython",major="3",minor="6",patchlevel="8",version="3.6.8"} 1.0
# HELP process_virtual_memory_bytes Virtual memory size in bytes.
# TYPE process_virtual_memory_bytes gauge
process_virtual_memory_bytes 3.76455168e+08
# HELP process_resident_memory_bytes Resident memory size in bytes.
# TYPE process_resident_memory_bytes gauge
process_resident_memory_bytes 1.6924672e+07
# HELP process_start_time_seconds Start time of the process since unix epoch in seconds.
# TYPE process_start_time_seconds gauge
process_start_time_seconds 1.59214640502e+09
# HELP process_cpu_seconds_total Total user and system CPU time spent in seconds.
# TYPE process_cpu_seconds_total counter
process_cpu_seconds_total 0.65
# HELP process_open_fds Number of open file descriptors.
# TYPE process_open_fds gauge
process_open_fds 7.0
# HELP process_max_fds Maximum number of open file descriptors.
# TYPE process_max_fds gauge
process_max_fds 1024.0
# HELP file_count Filescount in Path
# TYPE file_count gauge
Any help would be really helpful! i have also changed the folder/file permissions to 777 on all areas incase that this was the issue.
Thanks.