Variable Substitution call keys

[Gribouillis]

this should be easy.

You think it should be easy because you don't understand Python. Python is not a macro processor like Kernighan and Ritchie's m4 language. It is a programming language with a rich collection of types. For example the dict.keys() and dict.values() methods don't return strings. They return objects of type dict_keys and dict_values respectively, so your attempt to write

keys_string = new_test_dict['PCFData']['Q_STATISTICS_DATA']['SYSTEM.ADMIN.COMMAND.QUEUE'].keys()

does not work as you expect because keys_string is not a string.

You could try something, but it is ugly code which I would not recommend:

from ast import literal_eval
d = literal_eval(f'new_test_dict{pathto}')
keys = list(d.keys())
values = list(d.values())

The correct way as @bowlofred said above would be to parse patho to extract a Python list

['PCFData', 'Q_STATISTICS_DATA', 'SYSTEM.ADMIN.COMMAND.QUEUE']

then write one or two functions to follow the path in new_test_dict and its inner dictionaries.

[Bobbee]

Thanks everyone for the suggestions. I did get it working using eval(). I have not cleaned up the code. I need to add RESET_STATS to get the ENQ and DEQ rates on the queue in question. Then pump it to a CSV. I have all that code in another PY file. Here is what I ended up with.

#
##
### find all the Q_NAME values which is in the lowest nested dictionary(s)
##
#

QNAME_LIST = list(findkeys(new_test_dict, 'Q_NAME'))


##
## This will call recurse_object() to get the path to Q_NAME(s).
## This will allow us to get to that nested dictionary and get the path to it
##
for x in QNAME_LIST:
  print("----Q_NAME_LIST = ", x)
 
  recurse_object(new_test_dict, x)  # returns patho, path to nested dict
  
  outstring = ''
  count=patho.count(']')
  
  x = patho.split("]", count-1)
  count = count - 1
  
  for qualifiers in x:
  	if count > 0:
  	  outstring = outstring  + qualifiers + "]"
  	  count = count - 1
  
  fullpathkeys = "new_test_dict" + str(outstring) + ".keys()" + "\n"
  fullpathvalues = "new_test_dict" + str(outstring) + ".values()" + "\n"
  
  keys_string = eval(fullpathkeys)
  value_string = eval(fullpathvalues)
  
  keys_string_print = ",".join(list(keys_string))

  print("----- string of keys = ", keys_string_print)

  
keys_csv = ''
  
for q_string in keys_string:
	keys_csv = keys_csv + q_string + ','
	
values_csv = ''
  
for q_value in value_string:
  typeit = str(type(q_value))
  if "int" in typeit.lower():
     val = str(q_value)
  elif "str" in typeit.lower():
     val = q_value
  elif "list" in typeit.lower():
     total = 0
     for listvalue in q_value:
       total = total + int(listvalue)
     val = str(total)	  
  else:
     raise ValueError("Unknown data type:", typeit.lower())
  values_csv = values_csv + str(val) + ','

[deanhystad]

This is not the intended use for eval.

You did not post find_keys() or recurse_object(), so I am guessing here, but I might write like this:

data = {
    "Q_MGR_NAME": "BOBBEE2",
    "START_DATE": "2024-08-21",
    "START_TIME": "05.08.47",
    "END_DATE": "2024-08-21",
    "END_TIME": "05.09.47",
    "COMMAND_LEVEL": 940,
    "OBJECT_COUNT": 2,
    "Q_STATISTICS_DATA": {
        "SYSTEM.ADMIN.COMMAND.QUEUE": {
            "Q_NAME": "SYSTEM.ADMIN.COMMAND.QUEUE",
            "CREATION_DATE": "2023-04-18",
            "CREATION_TIME": "12.32.19",
            "Q_TYPE": "LOCAL",
            "DEFINITION_TYPE": "PREDEFINED",
            "Q_MIN_DEPTH": 0,
            "Q_MAX_DEPTH": 0,
            "AVG_Q_TIME": [0, 0],
            "PUTS": [17, 0],
            "PUTS_FAILED": 0,
            "PUT1S": [0, 0],
            "PUT1S_FAILED": 0,
            "PUT_BYTES": [1512, 0],
            "GETS": [17, 0],
            "GET_BYTES": [1512, 0],
            "GETS_FAILED": 0,
            "BROWSES": [0, 0],
            "BROWSE_BYTES": [0, 0],
            "BROWSES_FAILED": 0,
            "MSGS_NOT_QUEUED": 17,
            "MSGS_EXPIRED": 0,
            "MSGS_PURGED": 0,
        },
        "PYMQI": {
            "Q_NAME": "PYMQI",
            "CREATION_DATE": "2024-08-14",
            "CREATION_TIME": "07.39.31",
            "Q_TYPE": "LOCAL",
            "DEFINITION_TYPE": "PREDEFINED",
            "Q_MIN_DEPTH": 40,
            "Q_MAX_DEPTH": 60,
            "AVG_Q_TIME": [0, 0],
            "PUTS": [20, 0],
            "PUTS_FAILED": 0,
            "PUT1S": [0, 0],
            "PUT1S_FAILED": 0,
            "PUT_BYTES": [20480, 0],
            "GETS": [0, 0],
            "GET_BYTES": [0, 0],
            "GETS_FAILED": 0,
            "BROWSES": [0, 0],
            "BROWSE_BYTES": [0, 0],
            "BROWSES_FAILED": 0,
            "MSGS_NOT_QUEUED": 0,
            "MSGS_EXPIRED": 0,
            "MSGS_PURGED": 0,
        },
        "A_TEST": [1, 2, 3, {"Q_NAME": "IN_LIST_TEST", "CREATION_DATE": "2024-08-14"}, None],
    },
}


def find_dictionaries_with_key(dictionary, key):
    """Returns a list of dictionaries that contains key."""
    matches = []

    def worker(dictionary):
        if key in dictionary:
            matches.append(dictionary)
        for value in dictionary.values():
            if isinstance(value, dict):
                worker(value)
            elif isinstance(value, (list, tuple)):
                for item in value:
                    if isinstance(item, dict):
                        worker(item)

    worker(dictionary)
    return matches


def get_keys_and_values(dictionary):
    """Return dictionary keys and values as csv strings."""
    keys = []
    values = []
    for key, value in dictionary.items():
        if isinstance(value, (list, int, str)):
            keys.append(key)
            values.append(str(value))
    return ",".join(keys), ",".join(values)


for dictionary in find_dictionaries_with_key(data, "Q_NAME"):
    keys, values = get_keys_and_values(dictionary)
    print("Name:", dictionary["Q_NAME"])
    print(keys)
    print()

Output:
Name: SYSTEM.ADMIN.COMMAND.QUEUE
Q_NAME,CREATION_DATE,CREATION_TIME,Q_TYPE,DEFINITION_TYPE,Q_MIN_DEPTH,Q_MAX_DEPTH,AVG_Q_TIME,PUTS,PUTS_FAILED,PUT1S,PUT1S_FAILED,PUT_BYTES,GETS,GET_BYTES,GETS_FAILED,BROWSES,BROWSE_BYTES,BROWSES_FA,GETS_FAILED,BROWSES,BROWSE_BYTES,BROWSES_FAILED,MSGS_NOT_QUEUED,MSGS_EXPIRED,MSGS_PURGED

Name: PYMQI
Q_NAME,CREATION_DATE,CREATION_TIME,Q_TYPE,DEFINITION_TYPE,Q_MIN_DEPTH,Q_MAX_DEPTH,AVG_Q_TIME,PUTS,PUTS_FAILED,PUT1S,PUT1S_FAILED,PUT_BYTES,GETS,GET_BYTES,GETS_FAILED,BROWSES,BROWSE_BYTES,BROWSES_FA,GETS_FAILED,BROWSES,BROWSE_BYTES,BROWSES_FAILED,MSGS_NOT_QUEUED,MSGS_EXPIRED,MSGS_PURGED

Name: IN_LIST_TEST
Q_NAME,CREATION_DATE

Or do it in a single pass.

def find_dictionaries_with_key(dictionary, key):
    """Return keys and values for dictionaries that contain key."""
    matches = []

    def get_keys_and_values(dictionary):
        """Return dictionary keys and values as csv strings."""
        keys = []
        values = []
        for key, value in dictionary.items():
            if isinstance(value, (list, int, str)):
                keys.append(key)
                values.append(str(value))
        return ",".join(keys), ",".join(values)

    def walker(dictionary):
        """Walk through dictionary looking for dictionary that contains key"""
        if key in dictionary:
            matches.append(get_keys_and_values(dictionary))
        for value in dictionary.values():
            if isinstance(value, dict):
                walker(value)
            elif isinstance(value, (list, tuple)):
                for item in value:
                    if isinstance(item, dict):
                        walker(item)

    walker(dictionary)
    return matches


for keys, values in find_dictionaries_with_key(data, "Q_NAME"):
    print("Keys:", keys)
    print("Values:", values)
    print()

Output:
Keys: Q_NAME,CREATION_DATE,CREATION_TIME,Q_TYPE,DEFINITION_TYPE,Q_MIN_DEPTH,Q_MAX_DEPTH,AVG_Q_TIME,PUTS,PUTS_FAILED,PUT1S,PUT1S_FAILED,PUT_BYTES,GETS,GETbox/junk3.py_BYTES,GETS_FAILED,BROWSES,BROWSE_BYTES,BROWSES_FAILED,MSGS_NOT_QUEUED,MSGS_EXPIRED,MSGS_PURGED                                                          _BYTES,GETS_FAILED,BROWSES,BROWSE_BYTES,BROW
Values: SYSTEM.ADMIN.COMMAND.QUEUE,2023-04-18,12.32.19,LOCAL,PREDEFINED,0,0,[0, 0],[17, 0],0,[0, 0],0,[1512, 0],[17, 0],[1512, 0],0,[0, 0],[0, 0],0,17,0,0                                                                                                                                                        0

Keys: Q_NAME,CREATION_DATE,CREATION_TIME,Q_TYPE,DEFINITION_TYPE,Q_MIN_DEPTH,Q_MAX_DEPTH,AVG_Q_TIME,PUTS,PUTS_FAILED,PUT1S,PUT1S_FAILED,PUT_BYTES,GETS,GET_BYTES,GETS_FAILED,BROWSES,BROWSE_BYTES,BROW_BYTES,GETS_FAILED,BROWSES,BROWSE_BYTES,BROWSES_FAILED,MSGS_NOT_QUEUED,MSGS_EXPIRED,MSGS_PURGED
Values: PYMQI,2024-08-14,07.39.31,LOCAL,PREDEFINED,40,60,[0, 0],[20, 0],0,[0, 0],0,[20480, 0],[0, 0],[0, 0],0,[0, 0],[0, 0],0,0,0,0

Keys: Q_NAME,CREATION_DATE
Values: IN_LIST_TEST,2024-08-14

[Bobbee]

def findkeys(node, kv):
    if isinstance(node, list):
        for i in node:
            for x in findkeys(i, kv):
               yield x
    elif isinstance(node, dict):
        if kv in node:
            yield node[kv]
        for j in node.values():
            for x in findkeys(j, kv):
                yield x
                

###
####
#####
## This routine will parse through a given DICTIONARY and search for a given key.
## It will return the PATH to the KEY. this is used to PATH to a KEY in a nested
## dictionary.
#####
####
###
def recurse_object(obj_to_inspect, val_to_find, indexpath=''):
    global patho 
    if isinstance(obj_to_inspect, dict):
        for key, value in obj_to_inspect.items():
            recurse_object(value, val_to_find, indexpath + f"['{key}']")
    if isinstance(obj_to_inspect, list):
        for key, value in enumerate(obj_to_inspect):
            recurse_object(value, val_to_find, indexpath + f"[{key}]")
    if isinstance(obj_to_inspect, str):
        if obj_to_inspect == val_to_find:
            print(f"Value {val_to_find} found at {indexpath}")
            patho=indexpath
            return patho
            
###
####
#####
## 
## 
## 
#####
####
###
def find(element, JSON):     
  paths = element.split(".")  
  # print JSON[paths[0]][paths[1]][paths[2]][paths[3]]
  for i in range(0,len(paths)):
    data = JSON[paths[i]]
    # data = data[paths[i+1]]
    print(data)

[deanhystad]

find_keys() returns a list of dictionaries. recurse_object() gives you a list of keys, as a string, that get you from the top level dictionary to the nested dictionary. Is it clear that you have no need for recurse_object() and don't need to find a way to use patho to get the dictionaries returned by find_keys()? Just use the dictionaries returned by frind_keys().

And take a look at PEP8 that describes python coding conventions.

https://peps.python.org/pep-0008/

[Bobbee]

As requested, here is the full code, with ALL the debugging print statements removed. I program PYTHON occasionally when required. I pieced this together given the output from a package named 'mqtools' which extracts MQ PCF EVENT message from MQ. I needed to get that into CSV and into a daily report file to track performance. So I started and processed one step at a time. Again, everyone, thanks for your suggestions.

"""
Program to Test PYTHON Functions

Input parameters:
Dictionary

Output:
Queue Name
"""


import io
import codecs
import json
import pickle
import pymqi
import mqtools.mqpcf as MQPCF
import datetime
import argparse
import sys
import getpass
import struct
import string
import mqtools.MQ as MQ # for formatMQMD
from collections import OrderedDict # the data appears to contain an OrderedDict. 
global indexpath
from nested_lookup import nested_lookup

## 
###
####
###
##
def recursive_items(dictionary):
    for key, value in dictionary.items():
        if type(value) is dict:
            yield (key, value)
            yield from recursive_items(value)
        else:
            yield (key, value)
            
###
####
#####
## This routine will parse through a given DICTIONARY with nested dictionaries.
## It will display the keys and values of ALL keys.
#####
####
###
def findkeys(node, kv):
    if isinstance(node, list):
        for i in node:
            for x in findkeys(i, kv):
               yield x
    elif isinstance(node, dict):
        if kv in node:
            yield node[kv]
        for j in node.values():
            for x in findkeys(j, kv):
                yield x
                

###
####
#####
## This routine will parse through a given DICTIONARY and search for a given key.
## It will return the PATH to the KEY. this is used to PATH to a KEY in a nested
## dictionary.
#####
####
###
def recurse_object(obj_to_inspect, val_to_find, indexpath=''):
    global patho 
    if isinstance(obj_to_inspect, dict):
        for key, value in obj_to_inspect.items():
            recurse_object(value, val_to_find, indexpath + f"['{key}']")
    if isinstance(obj_to_inspect, list):
        for key, value in enumerate(obj_to_inspect):
            recurse_object(value, val_to_find, indexpath + f"[{key}]")
    if isinstance(obj_to_inspect, str):
        if obj_to_inspect == val_to_find:
            print(f"Value {val_to_find} found at {indexpath}")
            patho=indexpath
            return patho
            
 
## 
###
####
###
##
def find(element, JSON):     
  paths = element.split(".")  
  # print JSON[paths[0]][paths[1]][paths[2]][paths[3]]
  for i in range(0,len(paths)):
    data = JSON[paths[i]]
    # data = data[paths[i+1]]
    print(data)
 
###
####
######
# Python3 code to Read the serailzed dictionary created by get_pcf.py routine of the mqtools installment.
# the code will load the dictionary of MQPCF Statistics Events. It will isolate non-SYSTEM queues and
# write the results in CSV format to a file after adding DEQ and ENQ Reset Stats metrics to the list
# of values. this report is a daily report that is appended to everytime the routine is run.
#
# It is expected MQ Stats are set up on a Queue Level for targeted queues at a refesh rate selected by the 
# user. The script is used on the SYSTEM.ADMIN.STATISTICS.QUEUE using Triggering to process messages as 
# they appear.
#
# The CSV file can be imported into the provided EXCEL sheet. the Queue can be selected from the Q-NAME
# drop down which will refresh the chart on the second worksheet to show Queue Depth and the DEQ/ENQ
# rates.
#####
####
###                                                                Bobbee Broderick - [email protected]

# Input stream
mqpcf_stats = sys.stdin.readline()
new_test_dict = json.loads(mqpcf_stats)
 
    
###
### open the serailzed dictionary file from get_pcf.py
###
    
keys_list = list(new_test_dict.keys())
 
for key in new_test_dict:
    print(key)
    
for key, value in recursive_items(new_test_dict):
     if not isinstance(value, dict):
       print("Out of Recursive routine KEY =", key, "VALUE = ", value)
       print("Out of Recursive routine VALUE type", type(value))
       
QNAME_LIST = list(findkeys(new_test_dict, 'Q_NAME'))
 
##
## This will call recurse_object() to get the path to Q_NAME(s).
## This will allow us to get the keys and values for that queue
##
for x in QNAME_LIST:
  recurse_object(new_test_dict, x)
  
  outstring = ''
  count=patho.count(']')
  x = patho.split("]", count-1)
  count = count - 1
  
  for qualifiers in x:
  	if count > 0:
  	  outstring = outstring  + qualifiers + "]"
  	  count = count - 1
  
  fullpathkeys = "new_test_dict" + str(outstring) + ".keys()" + "\n"
  fullpathvalues = "new_test_dict" + str(outstring) + ".values()" + "\n"
  keys_string = eval(fullpathkeys)
  value_string = eval(fullpathvalues)
  keys_string_print = ",".join(list(keys_string))

  
keys_csv = ''
  
for q_string in keys_string:
	keys_csv = keys_csv + q_string + ','
	
values_csv = ''
  
for q_value in value_string:
  typeit = str(type(q_value))
  if "int" in typeit.lower():
     val = str(q_value)
  elif "str" in typeit.lower():
     val = q_value
  elif "list" in typeit.lower():
     total = 0
     for listvalue in q_value:
       total = total + int(listvalue)
     val = str(total)	  
  else:
     raise ValueError("Unknown data type:", typeit.lower())
  values_csv = values_csv + str(val) + ','