I write a lot of things down to save myself time the next time I need to do the same sort of thing — and publish this to the Internet in case I can save someone else time too. But this one is so specific, I’m not sure it’s an “ever going to encounter this again” sort of thing. Just in case, though — I have device data being stored in redis — because the device doesn’t know its throughput values, you need the last time and last value paired with the current device metrics to calculate throughput. OK. But, sporadically, the cached data is updated insomuch as a new record is posted with a new timestamp. But the actual values, other than timestamp, remain unchanged. With millions of interfaces, it’s challenging to identify these situations by spot-checking the visualizations. Instead, I need to monitor redis and identify when the tstamp is updated but no other values change.
import redis
import time
import re
import json
import os
# Configuration
redis_host = 'redishost.example.net'
redis_port = 6379
redis_password = 'P@5sw0rDG03sH3r3' # Replace with your Redis password
pattern = re.compile(r'INTERFACE_RAW_STATS_hostname\d\d\d\d_\d+_\d+')
output_file = 'changed_records.json'
# Connect to Redis
client = redis.StrictRedis(host=redis_host, port=redis_port, password=redis_password, decode_responses=True)
# Dictionary to track records
records = {}
matching_keys = []
def get_matching_keys():
"""
Retrieve keys from Redis matching the specified pattern.
Returns:
list: A list of keys that match the pattern.
"""
all_keys = client.keys()
matching_keys = [key for key in all_keys if pattern.match(key)]
return matching_keys
def process_keys():
"""
Process Redis keys to track changes in data.
Retrieves keys matching the pattern, gets their data using HGETALL,
and tracks changes. If only the 'tstamp' field has changed and all
other fields remain the same, the record is written to a file.
"""
global records
i = 0
for key in matching_keys:
i += 1
data = client.hgetall(key)
if i == 1 or i % 1000 == 0:
print(f"Processed {i} records")
if not data:
continue
collector_name = data.get('collectorName')
node_id = data.get('nodeId')
if_index = data.get('ifIndex')
tstamp = data.get('tstamp')
if not collector_name or not node_id or not if_index or not tstamp:
continue
unique_key = f"{collector_name}_{node_id}_{if_index}"
if unique_key in records:
previous_data = records[unique_key]
if previous_data['tstamp'] != tstamp:
# Check if all other values are the same
if all(data[k] == previous_data[k] for k in data if k != 'tstamp'):
print(f"***** Record changed: {json.dumps(data, indent=2)} *****")
write_to_file(data)
records[unique_key] = data # Update the record
else:
records[unique_key] = data
def write_to_file(data):
"""
Write the given data to a file.
Args:
data (dict): The data to write to the file.
"""
with open(output_file, 'a') as file:
file.write(json.dumps(data) + '\n')
if __name__ == "__main__":
# Ensure the output file is empty at the start
if os.path.exists(output_file):
os.remove(output_file)
# Retrieve the list of matching keys once
matching_keys = get_matching_keys()
while True:
process_keys()
print("Sleeping ... ")
time.sleep(300) # Sleep for 5 minutes