remove plugin in main munin distribution (jmx), and extract jmx2munin

plugins/java/jmx2munin/.gitignore

@@ -0,0 +1,8 @@
+.DS_Store
+.classpath
+.project
+.fatjar
+target
+eclipse
+old
+bin

plugins/java/jmx2munin/README.md

@@ -0,0 +1,79 @@
+# jmx2munin
+
+The [jmx2munin](http://github.com/tcurdt/jmx2munin) project exposes JMX MBean attributes to [Munin](http://munin-monitoring.org/).
+Some of it's features:
+
+ * strictly complies to the plugin format
+ * exposes composite types like Lists, Maps, Set as useful as possible
+ * String values can be mapped to numbers
+
+# How to use
+
+This is what the Munin script will call. So you should test this first. Of course with your parameters. This example expose all Cassandra information to Munin.
+
+    java -jar jmx2munin.jar \
+         -url service:jmx:rmi:///jndi/rmi://localhost:8080/jmxrmi \
+         -query "org.apache.cassandra.*:*"
+
+The "url" parameters specifies the JMX URL, the query selects the MBeans (and optionally also the attributes) to expose.
+
+    java -jar jmx2munin.jar \
+         -url service:jmx:rmi:///jndi/rmi://localhost:8080/jmxrmi \
+         -query "org.apache.cassandra.*:*" \
+         -attribute org_apache_cassandra_db_storageservice_livenodes_size
+
+The script that does the actual interaction with munin you can find in the contrib section. It's the one you should link in the your Munin plugin directory.
+
+    :/etc/munin/plugins$ ls -la cassandra_*
+    lrwxrwxrwx 1 root root 37 2011-04-07 19:58 cassandra_nodes_in_cluster -> /usr/share/munin/plugins/jmx2munin.sh
+
+In the plugin conf you point to the correct configuration
+
+    [cassandra_*]
+    env.query org.apache.cassandra.*:*
+
+    [cassandra_nodes_in_cluster]
+    env.config cassandra/nodes_in_cluster
+
+A possible configuration could look like this
+
+    graph_title Number of Nodes in Cluster
+    graph_vlabel org_apache_cassandra_db_storageservice_livenodes_size
+    org_apache_cassandra_db_storageservice_livenodes_size.label number of nodes
+
+The script will extract the attributes from the config and caches the JMX results to reduce the load when showing many values.
+
+# More advanced
+
+Sometimes it can be useful to track String values by mapping them into an enum as they really describe states. To find this possible candidates you can call:
+
+    java -jar jmx2munin.jar \
+         -url service:jmx:rmi:///jndi/rmi://localhost:8080/jmxrmi \
+         -query "org.apache.cassandra.*:*" \
+         list
+
+It should output a list of possible candidates. This can now be turned into a enum configuration file:
+
+    [org.apache.cassandra.db.StorageService:OperationMode]
+    0 = ^Normal
+    1 = ^Client
+    2 = ^Joining
+    3 = ^Bootstrapping
+    4 = ^Leaving
+    5 = ^Decommissioned
+    6 = ^Starting drain
+    7 = ^Node is drained
+
+Which you then can provide:
+
+    java -jar jmx2munin.jar \
+         -url service:jmx:rmi:///jndi/rmi://localhost:8080/jmxrmi \
+         -query "org.apache.cassandra.*:*" \
+         -enums /path/to/enums.cfg
+
+Now matching values get replaced by their numerical representation. On the left needs to be a unique number on the right side is a regular expression. If a string cannot be matched according to the spec "U" for "undefined" will be returned.
+
+# License
+
+Licensed under the Apache License, Version 2.0 (the "License")
+You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0

plugins/java/jmx2munin/contrib/jmx2munin.cfg/cassandra/nodes_in_cluster

@@ -0,0 +1,3 @@
+graph_title Number of Nodes in Cluster
+graph_vlabel org_apache_cassandra_db_storageservice_livenodes_size
+org_apache_cassandra_db_storageservice_livenodes_size.label number of nodes

plugins/java/jmx2munin/contrib/jmx2munin.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+# [cassandra_nodes_in_cluster]
+# env.config cassandra/nodes_in_cluster
+# env.query org.apache.cassandra.*:*
+
+if [ -z "$MUNIN_LIBDIR" ]; then
+    MUNIN_LIBDIR="`dirname $(dirname "$0")`"
+fi
+
+if [ -f "$MUNIN_LIBDIR/plugins/plugin.sh" ]; then
+    . $MUNIN_LIBDIR/plugins/plugin.sh
+fi
+
+if [ "$1" = "autoconf" ]; then
+    echo yes
+    exit 0
+fi
+
+if [ -z "$url" ]; then
+  # this is very common so make it a default
+  url="service:jmx:rmi:///jndi/rmi://127.0.0.1:8080/jmxrmi"
+fi
+
+if [ -z "$config" -o -z "$query" -o -z "$url" ]; then
+  echo "Configuration needs attributes config, query and optinally url"
+  exit 1
+fi
+
+JMX2MUNIN_DIR="$MUNIN_LIBDIR/plugins"
+CONFIG="$JMX2MUNIN_DIR/jmx2munin.cfg/$config"
+
+if [ "$1" = "config" ]; then
+    cat "$CONFIG"
+    exit 0
+fi
+
+JAR="$JMX2MUNIN_DIR/jmx2munin.jar"
+CACHED="/tmp/jmx2munin"
+
+if test ! -f $CACHED || test `find "$CACHED" -mmin +2`; then
+
+    java -jar "$JAR" \
+      -url "$url" \
+      -query "$query" \
+      $ATTRIBUTES \
+      > $CACHED
+
+    echo "cached.value `date +%s`" >> $CACHED
+fi
+
+ATTRIBUTES=`awk '/\.label/ { gsub(/\.label/,""); print $1 }' $CONFIG`
+
+for ATTRIBUTE in $ATTRIBUTES; do
+  grep $ATTRIBUTE $CACHED
+done

plugins/java/jmx2munin/pom.xml

@@ -0,0 +1,121 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>org.vafer</groupId>
+    <artifactId>jmx2munin</artifactId>
+    <name>jmx2munin</name>
+    <version>1.0</version>
+    <description>
+      Munin plugin to access JMX information
+    </description>
+    <url>http://github.com/tcurdt/jmx2munin</url>
+
+    <developers>
+        <developer>
+            <id>tcurdt</id>
+            <name>Torsten Curdt</name>
+            <email>tcurdt at vafer.org</email>
+            <timezone>+1</timezone>
+        </developer>
+    </developers>
+
+    <licenses>
+        <license>
+            <name>Apache License 2</name>
+            <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+        </license>
+    </licenses>
+
+    <scm>
+        <connection>scm:git:git://github.com:tcurdt/jmx2munin.git</connection>
+        <developerConnection>scm:git:git://github.com:tcurdt/jmx2munin.git</developerConnection>
+        <url>http://github.com/tcurdt/jmx2munin/tree/master</url>
+    </scm>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.beust</groupId>
+            <artifactId>jcommander</artifactId>
+            <version>1.17</version>
+        </dependency>
+
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>4.5</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <source>1.5</source>
+                    <target>1.5</target>
+                    <encoding>UTF-8</encoding>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <configuration>
+                    <forkMode>never</forkMode>
+                    <includes>
+                        <include>**/*TestCase.java</include>
+                    </includes>
+                    <excludes>
+                        <exclude>**/Abstract*</exclude>
+                    </excludes>
+                    <testFailureIgnore>true</testFailureIgnore>
+                    <skip>false</skip>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-source-plugin</artifactId>
+                <version>2.1</version>
+                <configuration>
+                    <attach>true</attach>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>create-source-jar</id>
+                        <goals>
+                            <goal>jar-no-fork</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>1.4</version>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+                            <minimizeJar>false</minimizeJar>
+                            <artifactSet>
+                                <includes>
+                                    <include>com.beust:jcommander</include>
+                                </includes>
+                            </artifactSet>
+                            <transformers>
+                                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                                    <mainClass>org.vafer.jmx.munin.Munin</mainClass>
+                                </transformer>
+                            </transformers>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>

plugins/java/jmx2munin/src/main/java/org/vafer/jmx/Enums.java

@@ -0,0 +1,77 @@
+package org.vafer.jmx;
+
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.regex.Pattern;
+
+import javax.management.ObjectName;
+
+public final class Enums {
+
+    private TreeMap<String, LinkedHashMap<Integer, Pattern>> sections = new TreeMap<String, LinkedHashMap<Integer, Pattern>>();
+    
+    public boolean load(String filePath) throws IOException {
+        BufferedReader input = null;
+        LinkedHashMap<Integer, Pattern> section = new LinkedHashMap<Integer, Pattern>();
+        try {
+            input = new BufferedReader(new InputStreamReader(new FileInputStream(filePath)));
+            String line;
+            int linenr = 0;
+            while((line = input.readLine()) != null) {
+                linenr += 1;
+                line = line.trim();
+                if (line.startsWith("#")) {
+                    continue;
+                }
+                if (line.startsWith("[") && line.endsWith("]")) {
+                    // new section
+                    String id = line.substring(1, line.length() - 1);
+                    section = new LinkedHashMap<Integer, Pattern>();
+                    sections.put(id, section);
+                } else {
+                    String[] pair = line.split("=");
+                    if (pair.length == 2) {
+                        Integer number = Integer.parseInt(pair[0].trim());
+                        Pattern pattern = Pattern.compile(pair[1].trim());
+                        if (section.put(number, pattern) != null) {
+                            System.err.println("Line " + linenr + ": previous definitions of " + number);
+                        }
+                    }
+                }
+            }
+        } finally {
+            if (input != null) {
+                input.close();      
+            }
+        }
+        return false;
+    }
+
+    public static String id(ObjectName beanName, String attributeName) {
+        StringBuilder sb = new StringBuilder();
+        sb.append(beanName.getDomain());
+        sb.append('.');
+        sb.append(beanName.getKeyProperty("type"));
+        sb.append(':');
+        sb.append(attributeName);
+        return sb.toString();        
+    }
+    
+    public Number resolve(String id, String value) {
+        LinkedHashMap<Integer, Pattern> section = sections.get(id);
+        if (section == null) {
+            return null;
+        }
+        for(Map.Entry<Integer, Pattern> entry : section.entrySet()) {
+            if (entry.getValue().matcher(value).matches()) {
+                return entry.getKey();
+            }
+        }        
+        return null;
+    }
+}

plugins/java/jmx2munin/src/main/java/org/vafer/jmx/Filter.java

@@ -0,0 +1,9 @@
+package org.vafer.jmx;
+
+import javax.management.ObjectName;
+
+public interface Filter {
+
+    public boolean include(ObjectName bean, String attribute);
+
+}

plugins/java/jmx2munin/src/main/java/org/vafer/jmx/ListOutput.java

@@ -0,0 +1,26 @@
+package org.vafer.jmx;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import javax.management.ObjectName;
+
+public final class ListOutput implements Output {
+
+    private final Set<String> seen = new HashSet<String>();
+
+    public void output(ObjectName beanName, String attributeName, Object value) {
+        Value.flatten(beanName, attributeName, value, new Value.Listener() {
+            public void value(ObjectName beanName, String attributeName, String value) {
+                final String id = Enums.id(beanName, attributeName);
+                if (!seen.contains(id)) {
+                    System.out.println("[" + id + "]");
+                    seen.add(id);
+                }
+            }
+            public void value(ObjectName beanName, String attributeName, Number value) {
+            }
+        });
+    }
+
+}

plugins/java/jmx2munin/src/main/java/org/vafer/jmx/NoFilter.java

@@ -0,0 +1,10 @@
+package org.vafer.jmx;
+
+import javax.management.ObjectName;
+
+public final class NoFilter implements Filter {
+
+    public boolean include(ObjectName bean, String attribute) {
+        return true;
+    }
+}

plugins/java/jmx2munin/src/main/java/org/vafer/jmx/Output.java

@@ -0,0 +1,9 @@
+package org.vafer.jmx;
+
+import javax.management.ObjectName;
+
+public interface Output {
+
+    public void output(ObjectName beanName, String attributeName, Object value);
+
+}

plugins/java/jmx2munin/src/main/java/org/vafer/jmx/Query.java

@@ -0,0 +1,52 @@
+package org.vafer.jmx;
+
+import java.io.IOException;
+import java.util.Collection;
+
+import javax.management.AttributeNotFoundException;
+import javax.management.InstanceNotFoundException;
+import javax.management.IntrospectionException;
+import javax.management.MBeanAttributeInfo;
+import javax.management.MBeanException;
+import javax.management.MBeanInfo;
+import javax.management.MBeanServerConnection;
+import javax.management.MalformedObjectNameException;
+import javax.management.ObjectInstance;
+import javax.management.ObjectName;
+import javax.management.ReflectionException;
+import javax.management.remote.JMXConnector;
+import javax.management.remote.JMXConnectorFactory;
+import javax.management.remote.JMXServiceURL;
+
+public final class Query {
+    
+    public void run(String url, String expression, Filter filter, Output output) throws IOException, MalformedObjectNameException, InstanceNotFoundException, ReflectionException, IntrospectionException, AttributeNotFoundException, MBeanException {
+        JMXConnector connector = JMXConnectorFactory.connect(new JMXServiceURL(url));
+        MBeanServerConnection connection = connector.getMBeanServerConnection();
+        final Collection<ObjectInstance> mbeans = connection.queryMBeans(new ObjectName(expression), null);
+
+        for(ObjectInstance mbean : mbeans) {
+            final ObjectName mbeanName = mbean.getObjectName();
+            final MBeanInfo mbeanInfo = connection.getMBeanInfo(mbeanName);
+            final MBeanAttributeInfo[] attributes = mbeanInfo.getAttributes();
+            for (final MBeanAttributeInfo attribute : attributes) {
+                if (attribute.isReadable()) {
+                    if (filter.include(mbeanName, attribute.getName())) {
+                        final String attributeName = attribute.getName();
+                        try {
+                            output.output(
+                                    mbean.getObjectName(),
+                                    attributeName,
+                                    connection.getAttribute(mbeanName, attributeName)
+                                    );
+                        } catch(Exception e) {
+                            // System.err.println("Failed to read " + mbeanName + "." + attributeName);
+                        }                        
+                    }
+                }
+            }
+
+        }
+        connector.close();
+    }
+}

plugins/java/jmx2munin/src/main/java/org/vafer/jmx/Value.java

@@ -0,0 +1,52 @@
+package org.vafer.jmx;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import javax.management.ObjectName;
+
+public final class Value {
+
+    public interface Listener {
+        public void value(ObjectName beanName, String attributeName, String value);
+        public void value(ObjectName beanName, String attributeName, Number value);
+    }
+    
+    public static void flatten(ObjectName beanName, String attributeName, Object value, Listener listener) {
+        if (value instanceof Number) {
+
+            listener.value(beanName, attributeName, (Number) value);
+        
+        } else if (value instanceof String) {
+
+            listener.value(beanName, attributeName, (String) value);
+
+        } else if (value instanceof Set) {
+
+            final Set set = (Set) value;
+            flatten(beanName, attributeName + ".size", set.size(), listener);                
+            for(Object entry : set) {                
+                flatten(beanName, attributeName + "[" + entry + "]", 1, listener);
+            }
+
+        } else if (value instanceof List) {
+
+            final List list = (List)value;
+            listener.value(beanName, attributeName + ".size", list.size());                
+            for(int i = 0; i<list.size(); i++) {
+                flatten(beanName, attributeName + "[" + i + "]", list.get(i), listener);
+            }
+            
+        } else if (value instanceof Map) {
+
+            final Map<?,?> map = (Map<?,?>) value;
+            listener.value(beanName, attributeName + ".size", map.size());                
+            for(Map.Entry<?, ?> entry : map.entrySet()) {
+                flatten(beanName, attributeName + "[" + entry.getKey() + "]", entry.getValue(), listener);                
+            }            
+            
+        } else {
+            // System.err.println("Failed to convert " + beanName + "." + attributeName);            
+        }
+    }
+}

plugins/java/jmx2munin/src/main/java/org/vafer/jmx/munin/Munin.java

@@ -0,0 +1,67 @@
+package org.vafer.jmx.munin;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+
+import org.vafer.jmx.Enums;
+import org.vafer.jmx.Filter;
+import org.vafer.jmx.ListOutput;
+import org.vafer.jmx.NoFilter;
+import org.vafer.jmx.Query;
+
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+
+public final class Munin {
+
+    @Parameter(description = "")
+    private List<String> args = new ArrayList<String>();
+    
+    @Parameter(names = "-url", description = "jmx url", required = true)
+    private String url;
+
+    @Parameter(names = "-query", description = "query expression", required = true)
+    private String query;
+
+    @Parameter(names = "-enums", description = "file string to enum config")
+    private String enumsPath;
+
+    @Parameter(names = "-attribute", description = "attributes to return")
+    private List<String> attributes = new ArrayList<String>();
+
+    private void run() throws Exception {
+        final Filter filter;
+        if (attributes == null || attributes.isEmpty()) {
+            filter = new NoFilter();
+        } else {
+            filter = new MuninAttributesFilter(attributes);
+        }
+
+        final Enums enums = new Enums();
+        if (enumsPath != null) {
+            enums.load(enumsPath);
+        }
+        
+        final String cmd = args.toString().toLowerCase(Locale.US);
+        if ("[list]".equals(cmd)) {
+            new Query().run(url, query, filter, new ListOutput());
+        } else {
+            new Query().run(url, query, filter, new MuninOutput(enums));
+        }
+    }
+
+    public static void main(String[] args) throws Exception {
+        Munin m = new Munin();
+        
+        JCommander cli = new JCommander(m);
+        try {
+            cli.parse(args);            
+        } catch(Exception e) {
+            cli.usage();
+            System.exit(1);
+        }
+
+        m.run();
+    }
+}

plugins/java/jmx2munin/src/main/java/org/vafer/jmx/munin/MuninAttributesFilter.java

@@ -0,0 +1,24 @@
+package org.vafer.jmx.munin;
+
+import java.util.HashSet;
+import java.util.List;
+
+import javax.management.ObjectName;
+
+import org.vafer.jmx.Filter;
+
+public final class MuninAttributesFilter implements Filter {
+
+    private final HashSet<String> attributes = new HashSet<String>();
+
+    public MuninAttributesFilter(List<String> pAttributes) {
+        for (String attribute : pAttributes) {
+            attributes.add(attribute.trim().replaceAll("_size$", ""));
+        }
+    }
+
+    public boolean include(ObjectName bean, String attribute) {
+        return attributes.contains(MuninOutput.attributeName(bean, attribute));
+    }
+
+}

plugins/java/jmx2munin/src/main/java/org/vafer/jmx/munin/MuninOutput.java

@@ -0,0 +1,93 @@
+package org.vafer.jmx.munin;
+
+import java.text.NumberFormat;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Hashtable;
+import java.util.Locale;
+
+import javax.management.ObjectName;
+
+import org.vafer.jmx.Enums;
+import org.vafer.jmx.Output;
+import org.vafer.jmx.Value;
+
+public final class MuninOutput implements Output {
+
+    private final Enums enums;
+
+    public MuninOutput(Enums enums) {
+        this.enums = enums;
+    }
+
+    public static String attributeName(ObjectName bean, String attribute) {
+        StringBuilder sb = new StringBuilder();
+        sb.append(fieldname(beanString(bean)));
+        sb.append('_');
+        sb.append(fieldname(attribute));
+        return sb.toString().toLowerCase(Locale.US);        
+    }
+    
+    private static String fieldname(String s) {
+        return s.replaceAll("[^A-Za-z0-9]", "_");
+    }
+
+    private static String beanString(ObjectName beanName) {
+        StringBuilder sb = new StringBuilder();
+        sb.append(beanName.getDomain());
+
+        Hashtable<String, String> properties = beanName.getKeyPropertyList();
+
+        String keyspace = "keyspace";
+        if (properties.containsKey(keyspace)) {
+            sb.append('.');
+            sb.append(properties.get(keyspace));
+            properties.remove(keyspace);
+        }
+
+        String type = "type";
+        if (properties.containsKey(type)) {
+            sb.append('.');
+            sb.append(properties.get(type));
+            properties.remove(type);
+        }
+
+        ArrayList<String> keys = new ArrayList(properties.keySet());
+        Collections.sort(keys);
+        
+        for(String key : keys) {
+            sb.append('.');
+            sb.append(properties.get(key));
+        }
+        
+        return sb.toString();
+        // return beanName.getCanonicalName();
+    }
+    
+    public void output(ObjectName beanName, String attributeName, Object value) {
+        Value.flatten(beanName, attributeName, value, new Value.Listener() {
+            public void value(ObjectName beanName, String attributeName, String value) {
+                final Number v = enums.resolve(Enums.id(beanName, attributeName), value);
+                if (v != null) {
+                    value(beanName, attributeName, v);
+                } else {
+                    value(beanName, attributeName, Double.NaN);                    
+                }
+            }
+            public void value(ObjectName beanName, String attributeName, Number value) {
+                final String v;
+
+                if (Double.isNaN(value.doubleValue())) {
+                    v = "U";
+                } else {
+                    final NumberFormat f = NumberFormat.getInstance();
+                    f.setMaximumFractionDigits(2);
+                    f.setGroupingUsed(false);
+                    v = f.format(value);            
+                }
+                
+                System.out.println(attributeName(beanName, attributeName) + ".value " + v);
+            }
+        });        
+   }
+}