Hello,
I wrote a custom Python.d plugin that is kinda working, but I have some questions about optimisation / logic.
My plugin monitor RQ queue jobs (https://python-rq.org/)
My code is the following:
# -*- coding: utf-8 -*-
# Description: howto weather station netdata python.d module
# Author: Panagiotis Papaioannou (papajohn-uop)
# SPDX-License-Identifier: GPL-3.0-or-later
from bases.FrameworkServices.SimpleService import SimpleService
import random
import requests
NETDATA_UPDATE_EVERY=1
priority = 90000
ORDER = ['default',
'low',
'collector',
'collector_background',
'integration',
'integration_background',
'collector_sync_auth_rq_worker',
'collector_auth',
'alfa',
'bankin_transaction_update',
'custify',
'email_collect']
CHARTS = {}
for name in ORDER:
CHARTS[name] = {
"options": ["default", "Job number", "integer", "RQ QUEUES", "RQ_Queue.default", "line"],
"lines": [
[name + "_jobs"],
[name + "_workers"],
[name + "_finished_jobs"],
[name + "_started_jobs"],
[name + "_deferred_jobs"],
[name + "_failed_jobs"],
[name + "_scheduled_jobs"]
]
}
class Service(SimpleService):
def __init__(self, configuration=None, name=None):
SimpleService.__init__(self, configuration=configuration, name=name)
self.order = ORDER
self.definitions = CHARTS
#values to show at graphs
self.values=dict()
@staticmethod
def check():
return True
rq_data=dict()
def logMe(self,msg):
self.debug(msg)
def populate_data(self):
url = 'https://myurl.com'
response = requests.get(url)
data = response.json()
for queue in data['queues']:
name = queue['name']
self.rq_data[name] = {}
self.rq_data[name]['jobs'] = queue['jobs']
self.rq_data[name]['workers'] = queue['workers']
self.rq_data[name]['finished_jobs'] = queue['finished_jobs']
self.rq_data[name]['started_jobs'] = queue['started_jobs']
self.rq_data[name]['deferred_jobs'] = queue['deferred_jobs']
self.rq_data[name]['failed_jobs'] = queue['failed_jobs']
self.rq_data[name]['scheduled_jobs'] = queue['scheduled_jobs']
def get_data(self):
#The data dict is basically all the values to be represented
# The entries are in the format: { "dimension": value}
#And each "dimension" should belong to a chart.
data = dict()
self.populate_data()
for name in ORDER:
data[name + '_jobs'] = self.rq_data[name]["jobs"]
data[name + '_workers'] = self.rq_data[name]["workers"]
data[name + '_finished_jobs'] = self.rq_data[name]["finished_jobs"]
data[name + '_started_jobs'] = self.rq_data[name]["started_jobs"]
data[name + '_deferred_jobs'] = self.rq_data[name]["deferred_jobs"]
data[name + '_failed_jobs'] = self.rq_data[name]["failed_jobs"]
data[name + '_scheduled_jobs'] = self.rq_data[name]["scheduled_jobs"]
return data
Question is the following:
As you can see, I have 12 charts, each of them should have the same lines. I had to do a workaround so each charts has his own line names (for example data[name + '_jobs'] = self.rq_data[name]["jobs"]) because if I don’t do that, well data[‘jobs’] will have the same value for all charts …
How can I fix this issue ?
Here is the JSON I’m reading (if it helps):
{"queues": [{"name": "alfa", "jobs": 0, "oldest_job_timestamp": "-", "index": 0, "connection_kwargs": {"db": 0, "username": null, "password": null, "socket_timeout": null, "encoding": "utf-8", "encoding_errors": "strict", "decode_responses": false, "retry_on_timeout": false, "health_check_interval": 0, "client_name": null, "host": "redis", "port": 6379, "socket_connect_timeout": null, "socket_keepalive": null, "socket_keepalive_options": null}, "workers": 64, "finished_jobs": 171, "started_jobs": 3, "deferred_jobs": 0, "failed_jobs": 58, "scheduled_jobs": 0}, {"name": "bankin_transaction_update", "jobs": 0, "oldest_job_timestamp": "-", "index": 1, "connection_kwargs": {"db": 0, "username": null, "password": null, "socket_timeout": null, "encoding": "utf-8", "encoding_errors": "strict", "decode_responses": false, "retry_on_timeout": false, "health_check_interval": 0, "client_name": null, "host": "redis", "port": 6379, "socket_connect_timeout": null, "socket_keepalive": null, "socket_keepalive_options": null}, "workers": 12, "finished_jobs": 0, "started_jobs": 0, "deferred_jobs": 127, "failed_jobs": 198, "scheduled_jobs": 0}, {"name": "collector", "jobs": 0, "oldest_job_timestamp": "-", "index": 2, "connection_kwargs": {"db": 0, "username": null, "password": null, "socket_timeout": null, "encoding": "utf-8", "encoding_errors": "strict", "decode_responses": false, "retry_on_timeout": false, "health_check_interval": 0, "client_name": null, "host": "redis", "port": 6379, "socket_connect_timeout": null, "socket_keepalive": null, "socket_keepalive_options": null}, "workers": 4, "finished_jobs": 0, "started_jobs": 0, "deferred_jobs": 0, "failed_jobs": 0, "scheduled_jobs": 0}, {"name": "collector_auth", "jobs": 0, "oldest_job_timestamp": "-", "index": 3, "connection_kwargs": {"db": 0, "username": null, "password": null, "socket_timeout": null, "encoding": "utf-8", "encoding_errors": "strict", "decode_responses": false, "retry_on_timeout": false, "health_check_interval": 0, "client_name": null, "host": "redis", "port": 6379, "socket_connect_timeout": null, "socket_keepalive": null, "socket_keepalive_options": null}, "workers": 5, "finished_jobs": 0, "started_jobs": 0, "deferred_jobs": 0, "failed_jobs": 1, "scheduled_jobs": 0}, {"name": "collector_background", "jobs": 0, "oldest_job_timestamp": "-", "index": 4, "connection_kwargs": {"db": 0, "username": null, "password": null, "socket_timeout": null, "encoding": "utf-8", "encoding_errors": "strict", "decode_responses": false, "retry_on_timeout": false, "health_check_interval": 0, "client_name": null, "host": "redis", "port": 6379, "socket_connect_timeout": null, "socket_keepalive": null, "socket_keepalive_options": null}, "workers": 4, "finished_jobs": 0, "started_jobs": 1, "deferred_jobs": 0, "failed_jobs": 0, "scheduled_jobs": 0}, {"name": "collector_sync_auth_rq_worker", "jobs": 0, "oldest_job_timestamp": "-", "index": 5, "connection_kwargs": {"db": 0, "username": null, "password": null, "socket_timeout": null, "encoding": "utf-8", "encoding_errors": "strict", "decode_responses": false, "retry_on_timeout": false, "health_check_interval": 0, "client_name": null, "host": "redis", "port": 6379, "socket_connect_timeout": null, "socket_keepalive": null, "socket_keepalive_options": null}, "workers": 4, "finished_jobs": 0, "started_jobs": 0, "deferred_jobs": 1, "failed_jobs": 7, "scheduled_jobs": 0}, {"name": "custify", "jobs": 0, "oldest_job_timestamp": "-", "index": 6, "connection_kwargs": {"db": 0, "username": null, "password": null, "socket_timeout": null, "encoding": "utf-8", "encoding_errors": "strict", "decode_responses": false, "retry_on_timeout": false, "health_check_interval": 0, "client_name": null, "host": "redis", "port": 6379, "socket_connect_timeout": null, "socket_keepalive": null, "socket_keepalive_options": null}, "workers": 12, "finished_jobs": 55, "started_jobs": 0, "deferred_jobs": 0, "failed_jobs": 0, "scheduled_jobs": 0}, {"name": "default", "jobs": 0, "oldest_job_timestamp": "-", "index": 7, "connection_kwargs": {"db": 0, "username": null, "password": null, "socket_timeout": null, "encoding": "utf-8", "encoding_errors": "strict", "decode_responses": false, "retry_on_timeout": false, "health_check_interval": 0, "client_name": null, "host": "redis", "port": 6379, "socket_connect_timeout": null, "socket_keepalive": null, "socket_keepalive_options": null}, "workers": 12, "finished_jobs": 205, "started_jobs": 1, "deferred_jobs": 0, "failed_jobs": 73, "scheduled_jobs": 0}, {"name": "email_collect", "jobs": 0, "oldest_job_timestamp": "-", "index": 8, "connection_kwargs": {"db": 0, "username": null, "password": null, "socket_timeout": null, "encoding": "utf-8", "encoding_errors": "strict", "decode_responses": false, "retry_on_timeout": false, "health_check_interval": 0, "client_name": null, "host": "redis", "port": 6379, "socket_connect_timeout": null, "socket_keepalive": null, "socket_keepalive_options": null}, "workers": 12, "finished_jobs": 0, "started_jobs": 0, "deferred_jobs": 0, "failed_jobs": 0, "scheduled_jobs": 0}, {"name": "integration", "jobs": 0, "oldest_job_timestamp": "-", "index": 9, "connection_kwargs": {"db": 0, "username": null, "password": null, "socket_timeout": null, "encoding": "utf-8", "encoding_errors": "strict", "decode_responses": false, "retry_on_timeout": false, "health_check_interval": 0, "client_name": null, "host": "redis", "port": 6379, "socket_connect_timeout": null, "socket_keepalive": null, "socket_keepalive_options": null}, "workers": 4, "finished_jobs": 0, "started_jobs": 0, "deferred_jobs": 0, "failed_jobs": 4, "scheduled_jobs": 0}, {"name": "integration_background", "jobs": 0, "oldest_job_timestamp": "-", "index": 10, "connection_kwargs": {"db": 0, "username": null, "password": null, "socket_timeout": null, "encoding": "utf-8", "encoding_errors": "strict", "decode_responses": false, "retry_on_timeout": false, "health_check_interval": 0, "client_name": null, "host": "redis", "port": 6379, "socket_connect_timeout": null, "socket_keepalive": null, "socket_keepalive_options": null}, "workers": 4, "finished_jobs": 0, "started_jobs": 0, "deferred_jobs": 0, "failed_jobs": 0, "scheduled_jobs": 0}, {"name": "low", "jobs": 0, "oldest_job_timestamp": "-", "index": 11, "connection_kwargs": {"db": 0, "username": null, "password": null, "socket_timeout": null, "encoding": "utf-8", "encoding_errors": "strict", "decode_responses": false, "retry_on_timeout": false, "health_check_interval": 0, "client_name": null, "host": "redis", "port": 6379, "socket_connect_timeout": null, "socket_keepalive": null, "socket_keepalive_options": null}, "workers": 12, "finished_jobs": 270, "started_jobs": 0, "deferred_jobs": 0, "failed_jobs": 231, "scheduled_jobs": 0}]}
Thanks for your help
DeWaRs