Robot/contrib-munin
2
0
Fork 0
mirror of https://github.com/munin-monitoring/contrib.git synced 2018-11-08 00:59:34 +01:00
Code Issues Releases Wiki Activity

Commit includes: fix slash in url params, examples update, fix issue #1 and #2, qps graph uses request instead of avgRequestPerSecond, new 'memory' graph, deprecation of availableram parameter

Browse Source
This commit is contained in:
antonio 2013-08-27 21:48:51 +02:00
parent 538cdc9359
commit ef960abcf9
1 changed files with 125 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

204
plugins/solr/solr4_
View File

@ -20,40 +20,27 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE. # DEALINGS IN THE SOFTWARE.
# #
# Munin plugin for monitoring a multicore solr 4.* installation via mbean. # Solr 4.* munin graph plugin
# It calls: # Plugins configuration parameters:
# > http://localhost:8080/solr/admin/cores?action=STATUS&wt=json
# and
# > http://localhost:8080/solr/corename/admin/mbeans?stats=true&wt=json
# for each core to retrieve cores and data. Verify those urls on your instance.
# #
# Configuration parameters:
# [solr_*] # [solr_*]
# host_port <host:port> # env.host_port <host:port>
# qpshandler_<handlerlabel> <handlerpath> # env.url <default /solr>
# availableram <ramsize in bytes> # env.qpshandler_<handlerlabel> <handlerpath>
# #
# Example: # ex:
# host_port solrhost:8080 # env.host_port solrhost:8080
# qpshandler_select /select # env.url /solr
# availableram 3221225472 # env.qpshandler_select /select
# #
# Defined checks: # Install plugins:
# numdocs
# qps
# indexsize
# requesttimes
# documentcache
# fieldvaluecache
# filtercache
# queryresultcache
#
# Installation example:
# ln -s /usr/share/munin/plugins/solr_.py /etc/munin/plugins/solr_numdocs_core_1 # ln -s /usr/share/munin/plugins/solr_.py /etc/munin/plugins/solr_numdocs_core_1
# ln -s /usr/share/munin/plugins/solr_.py /etc/munin/plugins/solr_requesttimes_select # ln -s /usr/share/munin/plugins/solr_.py /etc/munin/plugins/solr_requesttimes_select
# ln -s /usr/share/munin/plugins/solr_.py /etc/munin/plugins/solr_qps_core_1_select # ln -s /usr/share/munin/plugins/solr_.py /etc/munin/plugins/solr_core_1_select
# ln -s /usr/share/munin/plugins/solr_.py /etc/munin/plugins/solr_indexsize
# ln -s /usr/share/munin/plugins/solr_.py /etc/munin/plugins/solr_memory
# #
# Source repo: https://github.com/averni/munin-solr
import sys import sys
import os import os
@ -72,7 +59,7 @@ def parse_params():
data = params['core'].rsplit('_', 1) data = params['core'].rsplit('_', 1)
handler = data.pop() handler = data.pop()
params['params'] = { params['params'] = {
'handler': os.environ.get('qpshandler_%s' % handler, '/select') 'handler': os.environ.get('qpshandler_%s' % handler, 'standard')
} }
if not data: if not data:
params['core'] = '' params['core'] = ''
@ -90,7 +77,7 @@ class CheckException(Exception):
class JSONReader: class JSONReader:
@classmethod @classmethod
def readValue(cls, struct, path): def readValue(cls, struct, path, convert = None):
if not path[0] in struct: if not path[0] in struct:
return -1 return -1
obj = struct[path[0]] obj = struct[path[0]]
@ -98,15 +85,18 @@ class JSONReader:
return -1 return -1
for k in path[1:]: for k in path[1:]:
obj = obj[k] obj = obj[k]
if convert:
return convert(obj)
return obj return obj
class SolrCoresAdmin: class SolrCoresAdmin:
def __init__(self, host): def __init__(self, host, solrurl):
self.host = host self.host = host
self.solrurl = solrurl
self.data = None self.data = None
def fetchcores(self): def fetchcores(self):
uri = "/solr/admin/cores?action=STATUS&wt=json" uri = os.path.join(self.solrurl, "admin/cores?action=STATUS&wt=json")
conn = httplib.HTTPConnection(self.host) conn = httplib.HTTPConnection(self.host)
conn.request("GET", uri) conn.request("GET", uri)
res = conn.getresponse() res = conn.getresponse()
@ -135,13 +125,14 @@ class SolrCoresAdmin:
return ret return ret
class SolrCoreMBean: class SolrCoreMBean:
def __init__(self, host, core): def __init__(self, host, solrurl, core):
self.host = host self.host = host
self.data = None self.data = None
self.core = core self.core = core
self.solrurl = solrurl
def _fetch(self): def _fetch(self):
uri = "/solr/%s/admin/mbeans?stats=true&wt=json" % self.core uri = os.path.join(self.solrurl, "%s/admin/mbeans?stats=true&wt=json" % self.core)
conn = httplib.HTTPConnection(self.host) conn = httplib.HTTPConnection(self.host)
conn.request("GET", uri) conn.request("GET", uri)
res = conn.getresponse() res = conn.getresponse()
@ -159,37 +150,59 @@ class SolrCoreMBean:
data[key] = el data[key] = el
else: else:
key = el key = el
self._fetchSystem()
def _read(self, path): def _fetchSystem(self):
uri = os.path.join(self.solrurl, "%s/admin/system?stats=true&wt=json" % self.core)
conn = httplib.HTTPConnection(self.host)
conn.request("GET", uri)
res = conn.getresponse()
data = res.read()
if res.status != 200:
raise CheckException("System fetch failed: %s\n%s" %( str(res.status), res.read()))
self.data['system'] = json.loads(data)
def _readInt(self, path):
return self._read(path, int)
def _readFloat(self, path):
return self._read(path, float)
def _read(self, path, convert = None):
if self.data is None: if self.data is None:
self._fetch() self._fetch()
return JSONReader.readValue(self.data, path) return JSONReader.readValue(self.data, path, convert)
def _readCache(self, cache): def _readCache(self, cache):
result = {} result = {}
for key in ['lookups', 'hits', 'inserts', 'evictions', 'hitratio']: for key, ftype in [('lookups', int), ('hits', int), ('inserts', int), ('evictions', int), ('hitratio', float)]:
path = ['solr-mbeans', 'CACHE', cache, 'stats', 'cumulative_%s' % key] path = ['solr-mbeans', 'CACHE', cache, 'stats', 'cumulative_%s' % key]
result[key] = self._read(path) result[key] = self._read(path, ftype)
result['size'] = self._read(['solr-mbeans', 'CACHE', cache, 'stats', 'size']) result['size'] = self._readInt(['solr-mbeans', 'CACHE', cache, 'stats', 'size'])
return result return result
def getCore(self): def getCore(self):
return self.core return self.core
def requestcount(self, handler):
path = ['solr-mbeans', 'QUERYHANDLER', handler, 'stats', 'requests']
return self._readInt(path)
def qps(self, handler): def qps(self, handler):
path = ['solr-mbeans', 'QUERYHANDLER', handler, 'stats', 'avgRequestsPerSecond'] path = ['solr-mbeans', 'QUERYHANDLER', handler, 'stats', 'avgRequestsPerSecond']
return self._read(path) return self._readFloat(path)
def requesttimes(self, handler): def requesttimes(self, handler):
times = {} times = {}
path = ['solr-mbeans', 'QUERYHANDLER', handler, 'stats'] path = ['solr-mbeans', 'QUERYHANDLER', handler, 'stats']
for perc in ['avgTimePerRequest', '75thPcRequestTime', '99thPcRequestTime']: for perc in ['avgTimePerRequest', '75thPcRequestTime', '99thPcRequestTime']:
times[perc] = self._read(path + [perc]) times[perc] = self._read(path + [perc], float)
return times return times
def numdocs(self): def numdocs(self):
path = ['solr-mbeans', 'CORE', 'searcher', 'stats', 'numDocs'] path = ['solr-mbeans', 'CORE', 'searcher', 'stats', 'numDocs']
return self._read(path) return self._readInt(path)
def documentcache(self): def documentcache(self):
return self._readCache('documentCache') return self._readCache('documentCache')
@ -203,6 +216,13 @@ class SolrCoreMBean:
def queryresultcache(self): def queryresultcache(self):
return self._readCache('queryResultCache') return self._readCache('queryResultCache')
def memory(self):
data = self._read(['system', 'jvm', 'memory', 'raw'])
del data['used%']
for k in data.keys():
data[k] = int(data[k])
return data
############################################################################# #############################################################################
# Graph Templates # Graph Templates
@ -240,14 +260,19 @@ evictions.draw LINE2
""" """
QPSMAIN_GRAPH_TPL = """graph_title Solr {core} {handler} Request per second" QPSMAIN_GRAPH_TPL = """graph_title Solr {core} {handler} Request per second
graph_args -l 0 graph_args --base 1000 -r --lower-limit 0
graph_scale no
graph_vlabel request / second graph_vlabel request / second
graph_category solr graph_category solr
graph_period second
graph_order {gorder}
{cores_qps_graphs}""" {cores_qps_graphs}"""
QPSCORE_GRAPH_TPL = """qps_{core}.label {core} Request per second QPSCORE_GRAPH_TPL = """qps_{core}.label {core} Request per second
qps_{core}.type LINESTACK1 qps_{core}.draw {gtype}
qps_{core}.type DERIVE
qps_{core}.min 0
qps_{core}.graph yes""" qps_{core}.graph yes"""
REQUESTTIMES_GRAPH_TPL = """multigraph {core}_requesttimes REQUESTTIMES_GRAPH_TPL = """multigraph {core}_requesttimes
@ -256,15 +281,14 @@ graph_args -l 0
graph_vlabel millis graph_vlabel millis
graph_category solr graph_category solr
savgtimeperrequest_{core}.label {core} Avg time per request savgtimeperrequest_{core}.label {core} Avg time per request
savgtimeperrequest_{core}.type gauge savgtimeperrequest_{core}.type GAUGE
savgtimeperrequest_{core}.graph yes savgtimeperrequest_{core}.graph yes
s75thpcrequesttime_{core}.label {core} 75th perc s75thpcrequesttime_{core}.label {core} 75th perc
s75thpcrequesttime_{core}.type gauge s75thpcrequesttime_{core}.type GAUGE
s75thpcrequesttime_{core}.graph yes s75thpcrequesttime_{core}.graph yes
s99thpcrequesttime_{core}.label {core} 99th perc s99thpcrequesttime_{core}.label {core} 99th perc
s99thpcrequesttime_{core}.type gauge s99thpcrequesttime_{core}.type GAUGE
s99thpcrequesttime_{core}.graph yes s99thpcrequesttime_{core}.graph yes
""" """
NUMDOCS_GRAPH_TPL = """graph_title Solr Docs %s NUMDOCS_GRAPH_TPL = """graph_title Solr Docs %s
@ -272,39 +296,42 @@ graph_vlabel docs
docs.label Docs docs.label Docs
graph_category solr""" graph_category solr"""
INDEXSIZE_GRAPH_TPL = """graph_args --base 1024 -l 0 --upper-limit {availableram} INDEXSIZE_GRAPH_TPL = """graph_args --base 1024 -l 0
graph_vlabel Bytes graph_vlabel Bytes
graph_title Index Size graph_title Index Size
graph_category solr graph_category solr
graph_info Solr Index Memory Usage. graph_info Solr Index Size.
graph_order {cores} graph_order {cores}
{cores_config} {cores_config}
xmx.label Xmx
xmx.colour ff0000
""" """
INDEXSIZECORE_GRAPH_TPL = """{core}.label {core} INDEXSIZECORE_GRAPH_TPL = """{core}.label {core}
{core}.draw STACK""" {core}.draw STACK"""
MEMORYUSAGE_GRAPH_TPL = """graph_args --base 1024 -l 0 --upper-limit {availableram}
graph_vlabel Bytes
graph_title Solr memory usage
graph_category solr
graph_info Solr Memory Usage.
used.label Used
max.label Max
max.colour ff0000
"""
############################################################################# #############################################################################
# Graph managment # Graph managment
CHECKS_DEFINED = [
'numdocs',
'qps',
'indexsize',
'requesttimes',
'documentcache',
'fieldvaluecache',
'filtercache',
'queryresultcache'
]
class SolrMuninGraph: class SolrMuninGraph:
def __init__(self, hostport, solrmbean): def __init__(self, hostport, solrurl, params):
self.solrcoresadmin = SolrCoresAdmin(hostport) self.solrcoresadmin = SolrCoresAdmin(hostport, solrurl)
self.hostport = hostport self.hostport = hostport
self.solrurl = solrurl
self.params = params self.params = params
def _getMBean(self, core): def _getMBean(self, core):
return SolrCoreMBean(self.hostport, core) return SolrCoreMBean(self.hostport, self.solrurl, core)
def _cacheConfig(self, cacheType, cacheName): def _cacheConfig(self, cacheType, cacheName):
return CACHE_GRAPH_TPL.format(core=self.params['core'], cacheType=cacheType, cacheName=cacheName) return CACHE_GRAPH_TPL.format(core=self.params['core'], cacheType=cacheType, cacheName=cacheName)
@ -318,16 +345,14 @@ class SolrMuninGraph:
data = getattr(solrmbean, cacheType)() data = getattr(solrmbean, cacheType)()
results.append('multigraph solr_{core}_{cacheType}_hit_rates'.format(core=self.params['core'], cacheType=cacheType)) results.append('multigraph solr_{core}_{cacheType}_hit_rates'.format(core=self.params['core'], cacheType=cacheType))
for label in hits_fields: for label in hits_fields:
results.append("%s.value %s" % (label, data[label])) results.append("%s.value %.8f" % (label, data[label]))
results.append('multigraph solr_{core}_{cacheType}_size'.format(core=self.params['core'], cacheType=cacheType)) results.append('multigraph solr_{core}_{cacheType}_size'.format(core=self.params['core'], cacheType=cacheType))
for label in size_fields: for label in size_fields:
results.append("%s.value %s" % (label, data[label])) results.append("%s.value %d" % (label, data[label]))
return "\n".join(results) return "\n".join(results)
def config(self, mtype): def config(self, mtype):
if not mtype: if not mtype or not hasattr(self, '%sConfig' % mtype):
raise CheckException("""Check missing. Available checks: \n\t%s""" % '\n\t'.join(CHECKS_DEFINED))
if not hasattr(self, '%sConfig' % mtype):
raise CheckException("Unknown check %s" % mtype) raise CheckException("Unknown check %s" % mtype)
return getattr(self, '%sConfig' % mtype)() return getattr(self, '%sConfig' % mtype)()
@ -345,12 +370,13 @@ class SolrMuninGraph:
def qpsConfig(self): def qpsConfig(self):
cores = self._getCores() cores = self._getCores()
graph = [QPSCORE_GRAPH_TPL.format(core=c) for c in cores ] graph = [QPSCORE_GRAPH_TPL.format(core=c, gtype='LINESTACK1') for pos,c in enumerate(cores) ]
return QPSMAIN_GRAPH_TPL.format( return QPSMAIN_GRAPH_TPL.format(
cores_qps_graphs='\n'.join(graph), cores_qps_graphs='\n'.join(graph),
handler=self.params['params']['handler'], handler=self.params['params']['handler'],
core=self.params['core'], core=self.params['core'],
cores_qps_cdefs='%s,%s' % (','.join(map(lambda x: 'qps_%s' % x, cores)),','.join(['+']*(len(cores)-1))) cores_qps_cdefs='%s,%s' % (','.join(map(lambda x: 'qps_%s' % x, cores)),','.join(['+']*(len(cores)-1))),
gorder=','.join(cores)
) )
def qps(self): def qps(self):
@ -358,7 +384,7 @@ class SolrMuninGraph:
cores = self._getCores() cores = self._getCores()
for c in cores: for c in cores:
mbean = self._getMBean(c) mbean = self._getMBean(c)
results.append('qps_%s.value %s' % (c, mbean.qps(self.params['params']['handler']))) results.append('qps_%s.value %d' % (c, mbean.requestcount(self.params['params']['handler'])))
return '\n'.join(results) return '\n'.join(results)
def requesttimesConfig(self): def requesttimesConfig(self):
@ -373,7 +399,7 @@ class SolrMuninGraph:
mbean = self._getMBean(c) mbean = self._getMBean(c)
results.append('multigraph {core}_requesttimes'.format(core=c)) results.append('multigraph {core}_requesttimes'.format(core=c))
for k, time in mbean.requesttimes(self.params['params']['handler']).items(): for k, time in mbean.requesttimes(self.params['params']['handler']).items():
results.append('s%s_%s.value %s' % (k.lower(), c, time)) results.append('s%s_%s.value %.5f' % (k.lower(), c, time))
return '\n'.join(results) return '\n'.join(results)
def numdocsConfig(self): def numdocsConfig(self):
@ -381,20 +407,36 @@ class SolrMuninGraph:
def numdocs(self): def numdocs(self):
mbean = self._getMBean(self.params['core']) mbean = self._getMBean(self.params['core'])
return 'docs.value %s' % mbean.numdocs(**self.params['params']) return 'docs.value %d' % mbean.numdocs(**self.params['params'])
def indexsizeConfig(self): def indexsizeConfig(self):
cores = self._getCores() cores = self._getCores()
availableram = os.environ.get('availableram', 16868532224)
graph = [ INDEXSIZECORE_GRAPH_TPL.format(core=c) for c in cores] graph = [ INDEXSIZECORE_GRAPH_TPL.format(core=c) for c in cores]
return INDEXSIZE_GRAPH_TPL.format(cores=" ".join(cores), cores_config="\n".join(graph), availableram=availableram) return INDEXSIZE_GRAPH_TPL.format(cores=" ".join(cores), cores_config="\n".join(graph))
def indexsize(self): def indexsize(self):
results = [] results = []
for c, size in self.solrcoresadmin.indexsize(**self.params['params']).items(): for c, size in self.solrcoresadmin.indexsize(**self.params['params']).items():
results.append("%s.value %s" % (c, size)) results.append("%s.value %d" % (c, size))
cores = self._getCores()
mbean = self._getMBean(cores[0])
memory = mbean.memory()
results.append('xmx.value %d' % memory['max'])
return "\n".join(results) return "\n".join(results)
def memoryConfig(self):
cores = self._getCores()
mbean = self._getMBean(cores[0])
memory = mbean.memory()
return MEMORYUSAGE_GRAPH_TPL.format(availableram=memory['max'] * 1.05)
def memory(self):
results = []
cores = self._getCores()
mbean = self._getMBean(cores[0])
memory = mbean.memory()
return '\n'.join(['used.value %d' % memory['used'], 'max.value %d' % memory['max']])
def documentcacheConfig(self): def documentcacheConfig(self):
return self._cacheConfig('documentcache', 'Document Cache') return self._cacheConfig('documentcache', 'Document Cache')
@ -422,10 +464,10 @@ class SolrMuninGraph:
if __name__ == '__main__': if __name__ == '__main__':
params = parse_params() params = parse_params()
SOLR_HOST_PORT = os.environ.get('host_port', 'localhost:8080').replace('http://', '') SOLR_HOST_PORT = os.environ.get('host_port', 'localhost:8080').replace('http://', '')
mb = SolrMuninGraph(SOLR_HOST_PORT, params) SOLR_URL = os.environ.get('url', '/solr')
try: if SOLR_URL[0] != '/':
SOLR_URL = '/' + SOLR_URL
mb = SolrMuninGraph(SOLR_HOST_PORT, SOLR_URL, params)
if hasattr(mb, params['op']): if hasattr(mb, params['op']):
print getattr(mb, params['op'])(params['type']) print getattr(mb, params['op'])(params['type'])
except Exception, ex:
print "ERROR: %s" % ex
exit(1)