Massively Scalable Data Center (MSDC) Design and Implementation Guide
Buffer Monitoring Code and Configuration Files
Downloads: This chapterpdf (PDF - 246.0KB) The complete bookPDF (PDF - 6.82MB) | Feedback

Buffer Monitoring Code and Configuration Files

Table Of Contents

Buffer Monitoring Code and Configuration Files

buffer_check.py

check_process.py

NX-OS Scheduler Example

Collectd Configuration

collectd.conf

Puppet Manifest

Graphite Configuration

carbon.conf

graphite.wsgi

graphite-vhost.conf

local_settings.py

relay-rules.conf

storage-schemas.conf

Puppet Manifest (init.pp)


Buffer Monitoring Code and Configuration Files


The following buffer monitoring code and configuration files are available for consideration:

buffer_check.py

check_process.py

NX-OS Scheduler Example

Collectd Configuration

collectd.conf

Puppet Manifest

Graphite Configuration

carbon.conf

graphite.wsgi

graphite-vhost.conf

local_settings.py

relay-rules.conf

storage-schemas.conf

Puppet Manifest (init.pp)

buffer_check.py

#!/usr/bin/python
#
# A script for monitoring buffer utilization on the Cisco Nexus 3000
# platform. Tested with Nexus 3064 and Nexus 3048 switches. Intended
# to be run on the switch. Reports data to Graphite via pickled data
# over TCP (or any other data sink that can read pickle data).
#
# Written by Mark T. Voelker
# Copyright 2012 Cisco Systems, Inc.
#
 
   
import os
import sys
import re
import logging
import argparse
import time
import cPickle
import socket
import struct
import copy
import xml.parsers.expat
from cisco import CLI
 
   
 
   
def daemonize():
    """
    Daemonizes the process by forking the main execution off
    into the background.
    """
    try:
        pid = os.fork()
    except OSError, e:
        raise OSError("Can't fork(%d): %s" % (e.errno, e.strerror))
 
   
    if (pid == 0):
        # This is the child process.
        # Become the session leder/process group leader and ensure
        # that we don't have a controlling terminal.
        os.setsid()
 
   
        # Now do some work.
    else:
        # This is the parent.
        # Write the pid of the child before we quit.
        write_pidfile(pid)
        global logger
        logger.debug(
            "Parent (%d) spawned child (%d) successfully" % (os.getpid(), pid)
            )
        exit(0)
 
   
 
   
def write_pidfile(pid=os.getpid()):
    """
    Writes a pid file to /bootflash/buffer_check.py.pid.
    The file contains one line with the PID.
    """
    global args
    f = open(args.pidfile, 'w')
    f.write(str(pid))
    f.close()
 
   
 
   
def set_exit_code(value, current_code):
    """
    Returns an exit code taking into account any previous conditions
    that set an exit code.
    """
    if exit_code >= 2:
        # Nothing can change this.
        logger.debug("exit code is already set to 2")
        return 2
    else:
        logger.debug("exit code set to %s" % (current_code))
        return current_code
 
   
 
   
def start_element(name, attrs):
    """
    Callback routine to handle the start of a tag.
    """
    global current_tag
    current_tag = copy.copy(name)
    #logger.debug("Current tag: '%s'" % (current_tag))
 
   
 
   
def end_element(name):
    """
    Callback routine for handling the end of a tagged element.
    """
    global current_tag
    current_tag = ''
 
   
 
   
def char_data(data):
    """
    Callback routine to handle data within a tag.
    """
    global current_tag
    global current_int
    global parsed_data
    #logger.debug("char_data handler called [current_tag = %s] on '%s'" % (
    #    current_tag, data)
    #    )
    if current_tag == 'total_instant_usage':
        parsed_data['instant_cell_usage'] = int(copy.copy(data))
        logger.debug("FOUND TOTAL INSTANT CELL USAGE: %s" % (data))
    elif current_tag == 'max_cell_usage':
        parsed_data['max_cell_usage'] = int(copy.copy(data))
        logger.debug("FOUND TOTAL MAX CELL USAGE: %s" % (data))
    elif current_tag == 'rem_instant_usage':
        parsed_data['rem_instant_usage'] = int(copy.copy(data))
        logger.debug("FOUND REMAINING INSTANT USAGE: %s" % (data))
    elif current_tag == 'front_port':
        current_int = int(copy.copy(data))
        parsed_data[current_int] = 0
        logger.debug("Started a new front port: %s" % (data))
    elif re.search('^[m|u]cast_count_\d$', current_tag):
        logger.debug("Found queue counter (port %s): %s" % (current_int, data))
        if current_int in parsed_data:
            parsed_data[current_int] += int(copy.copy(data))
        else:
            parsed_data[current_int] = int(copy.copy(data))
        logger.debug("Added %s to counter for port %s (total: %s)" % (
            data, current_int, parsed_data[current_int])
            )
 
   
 
   
def int_char_data(data):
    """
    Callback routine to handle data within a tag.
    """
    global interface_rates
    global current_tag
    global current_int
    global get_cmd_timestamp
    global pickle_data
    global logger
 
   
    # List of the tags we care about.
    keepers = ['eth_outbytes', 'eth_inbytes', 'eth_outpkts', 'eth_inpkts']
 
   
    if current_tag in keepers:
        # Set up some data storage.
        logger.debug("Working on %s for %s..." % (current_tag, current_int))
        if current_int not in interface_rates:
            interface_rates[current_int] = dict()
            logger.debug("   allocating space for %s" % (current_int))
        if 'last' not in interface_rates[current_int]:
            interface_rates[current_int]['last'] = dict()
            interface_rates[current_int]['last']['timestamp'] = 0.0
            logger.debug("Initializing %s last timestamp to 0." % (
                current_int))
 
   
        # Before we start working on the data, go ahead and pickle
        # the raw counter stat.
        pickle_data.append(
            tuple(['iface_%s.1-%s.%s' % (
                current_tag, current_int, hostname),
                tuple([get_cmd_timestamp, data])]))
        logger.debug("Pickled iface_%s.1-%s.%s: %s" % (
               current_tag, current_int, hostname, data
               ))
 
   
        # Make sure we're set up to hold this data properly.
        if current_tag not in interface_rates[current_int]['last']:
            interface_rates[current_int]['last'][current_tag] = 0
 
   
        # Calculate the rates of change.
        logger.debug("Calculating rate for %s/%s using (%s-%s)/(%s-%s)" % (
            current_tag, current_int, data,
            interface_rates[current_int]['last'][current_tag],
            get_cmd_timestamp,
            interface_rates[current_int]['last']['timestamp']
            ))
        rate = (float(data) - \
            int(interface_rates[current_int]['last'][current_tag])) / \
            (get_cmd_timestamp - \
            interface_rates[current_int]['last']['timestamp'])
        pickle_data.append(
            tuple(['iface_%s_rate.1-%s.%s' % (
                current_tag, current_int, hostname),
                tuple([get_cmd_timestamp, rate])]))
        logger.debug("Pickled iface_%s_rate.1-%s.%s: %s" % (
           current_tag, current_int, hostname, rate
           ))
 
   
        # Per user request, we convert byte rates to bit rates.
        if re.search('bytes$', current_tag):
            logger.debug(
                "Calculating bitrate for %s/%s using (%s-%s)/(%s-%s)*8" % (
                current_tag, current_int, data,
                interface_rates[current_int]['last'][current_tag],
                get_cmd_timestamp,
                interface_rates[current_int]['last']['timestamp']
                ))
            rate = (float(data) - \
                int(interface_rates[current_int]['last'][current_tag])) / \
                (get_cmd_timestamp - \
                interface_rates[current_int]['last']['timestamp']) * 8
            bitname = copy.copy(current_tag)
            bitname = re.sub('bytes$', 'bits', bitname)
            pickle_data.append(
                tuple(['iface_%s_rate.1-%s.%s' % (
                    bitname, current_int, hostname),
                    tuple([get_cmd_timestamp, rate])]))
            logger.debug("Pickled bitrate iface_%s_rate.1-%s.%s: %s" % (
               bitname, current_int, hostname, rate
               ))
 
   
        # Now store the current data.
        interface_rates[current_int]['last'][current_tag] = \
            int(copy.copy(data))
 
   
 
   
def get_show_queuing_int():
    """
    Parses output from 'show queuing interface' and reports stats.
    Unicast drop stats are reported for each interface given in the
    list of interfaces on the command line.  Drop stats for multicast,
    unicast, xon, and xoff are added up for all interfaces (including
    those not specified on the command line) to provide switch-level
    totals for each.
 
   
    Note that there is no XML output for 'show queuing interface' at
    present, so we're forced to parse plaintext from the CLI.  XML
    output does exist for 'show queuing interface x/y | xml', however
    this would require issuing one command for each interface on the box
    since we need to provide switch-level totals.  As this would be
    a performance bottleneck due to the number of commands to be issued
    and parsed, we've avoided that approach here.
    """
    global pickle_data
    global logger
    global args
 
   
    logger.debug("Issuing 'show queuing interface' command...")
    get_cmd_timestamp = time.time()
    cli_obj = CLI('show queuing interface', False)
    cli_output = cli_obj.get_output()
 
   
    # As we parse, remember what interface we're working with.
    current_int = ''
 
   
    # Set up switch-level total counters.
    switch_counters = dict()
    switch_counters['ucast_pkts_dropped'] = 0
    switch_counters['ucast_bytes_dropped'] = 0
    switch_counters['mcast_pkts_dropped'] = 0
    switch_counters['mcast_bytes_dropped'] = 0
    switch_counters['xon'] = 0
    switch_counters['xoff'] = 0
 
   
    for line in cli_output:
        match = re.match('Ethernet(\d+\/\d+) queuing information', line)
        if match:
            current_int = match.group(1).replace('/', '-')
            logger.debug("Working on queuing stat for int %s" % (current_int))
            continue
        match = re.search('drop-type:\s+drop,\s+xon:\s*(\d+),\s+xoff:\s*(\d+)',
            line)
        if match:
            # As of this revision, we don't collect individual
            # interface counters per interface.
            switch_counters['xon'] += int(match.group(1))
            switch_counters['xoff'] += int(match.group(2))
            continue
        match = re.search('([UM]cast) (pkts|bytes) dropped\s+:\s*(\d+)', line)
        if match:
            stat_name = "%s_%s_dropped.%s.%s" % (
                match.group(1).lower(), match.group(2).lower(),
                current_int, hostname
                )
            switch_stat_name = "%s_%s_dropped" % (match.group(1).lower(),
                match.group(2).lower())
 
   
            # If it's a unicast stat and this interface is
            # in the list given in the CLI, pickle it.
            if re.match('ucast_', stat_name):
                int_on_lc = re.match('\d+\-(\d+)', current_int)
                int_on_lc = int(int_on_lc.group(1))
                if int_on_lc in args.interfaces:
                    pickle_data.append(
                        tuple([
                            stat_name,
                            tuple(
                                [get_cmd_timestamp,
                                int(match.group(3))])]))
                    logger.debug("Pickled %s: %s" % (stat_name,
                        match.group(3)))
 
   
            # Add to our switch-level counters.
            switch_counters[switch_stat_name] += int(match.group(3))
 
   
    # Output parsing complete...pickle the switch-level stats.
    for stat_name in switch_counters:
        pickle_data.append(
            tuple([
                stat_name + '.' + hostname,
                tuple([get_cmd_timestamp, switch_counters[stat_name]])]))
        logger.debug("Pickled %s.%s: %s" % (stat_name, hostname,
            switch_counters[stat_name]))
 
   
 
   
def get_int_counters():
    """
    Parses stats from the output of 'show interface x/y | xml'.
    """
    global args
    global pickle_data
    global logger
    global interface_rates
    global current_int
    global current_tag
    global get_cmd_timestamp
 
   
    # Sift through each interface.
    for port_num in args.interfaces:
        # Now handle any interface-specific counters for this port.
        try:
            current_int = port_num
            get_cmd_timestamp = time.time()
            cli_obj = CLI("show int e1/%s | xml" % (port_num), False)
 
   
            # Get the reply.
            get_cmd_reply = cli_obj.get_raw_output()
            logger.debug("-----\nReply received:\n-----" + str(get_cmd_reply))
 
   
            # Clean off trailing junk...is this an NX-OS bug?
            get_cmd_reply = get_cmd_reply.rstrip(">]\n") + '>'
 
   
            # Set up an XML parser and parse.
            int_xml_parser = xml.parsers.expat.ParserCreate()
            int_xml_parser.StartElementHandler = start_element
            int_xml_parser.EndElementHandler = end_element
            int_xml_parser.CharacterDataHandler = int_char_data
            int_xml_parser.Parse(get_cmd_reply, 1)
 
   
            # Remember the timestamp of this command for the next go around.
            if port_num not in interface_rates:
                interface_rates[port_num] = dict()
            if 'last' not in interface_rates[port_num]:
                interface_rates[port_num]['last'] = dict()
                logger.debug("Initializing %s last dict for " % (
                    port_num))
            interface_rates[port_num]['last']['timestamp'] = \
                copy.copy(get_cmd_timestamp)
            logger.debug("Set last timestamp for %s to %s" % (
                port_num, get_cmd_timestamp))
        except SyntaxError:
            print """
    WARNING: can't get output for interface 1/%s.  Does it exist?
    """ % (port_num)
 
   
 
   
def get_buffer_stats():
    """
    Parses stats from the output of 'show hardware internal buffer pkt-stats detail | 
xml'.
    """
    global args
    global pickle_data
    global logger
    global interface_rates
    global exit_code
 
   
    # Frame up the command snippet we need to send to the switch.
    get_message = "show hardware internal buffer info pkt-stats detail | xml"
 
   
    # Set up the CLI object and issue the command.
    get_cmd_timestamp = time.time()
    cli_obj = CLI(get_message, False)
 
   
    # Before we process the reply, send another message to clear the counters
    # unless we've been told not to do so.
    if args.clear_counters:
        clear_obj = CLI(clear_message)
        clear_cmd_reply = clear_obj.get_raw_output()
        logger.debug("Result of clear command:\n%s" % (clear_cmd_reply))
 
   
    # Parse the reply.
    get_cmd_reply = cli_obj.get_raw_output()
    logger.debug("-----\nReply received:\n-----" + str(get_cmd_reply))
 
   
    # Clean off trailing junk...is this an NX-OS bug?
    get_cmd_reply = get_cmd_reply.rstrip(">]\n") + '>'
 
   
    # Start up an expat parser to quickly grock the XML.
    xml_parser = xml.parsers.expat.ParserCreate()
    xml_parser.StartElementHandler = start_element
    xml_parser.EndElementHandler = end_element
    xml_parser.CharacterDataHandler = char_data
    xml_parser.Parse(get_cmd_reply, 1)
 
   
    # Form a pickle-protocol data structure.
    output_string = ""
 
   
    # Pickle max buffer usage if necessary.
    if args.get_max_buf:
        logger.debug("Max cell usage is %s" % (parsed_data['max_cell_usage']))
        output_string += "Max cell usage: %s" % (parsed_data['max_cell_usage'])
        pickle_data.append(tuple(['max_cell_usage.%s' % (hostname),
            tuple([get_cmd_timestamp, parsed_data['max_cell_usage']])])
            )
        exit_code = set_exit_code(
            int(parsed_data['max_cell_usage']), exit_code
            )
 
   
    # Now do instant cell usage.
    if args.get_instant_buf:
        logger.debug("Instant cell usage is %s" % (
            parsed_data['instant_cell_usage'])
            )
        if args.get_max_buf:
            output_string += ', '
        output_string += "Instant cell usage: %s" % (
            parsed_data['instant_cell_usage']
            )
        pickle_data.append(tuple(['instant_cell_usage.%s' % (hostname),
            tuple([get_cmd_timestamp, parsed_data['instant_cell_usage']])])
            )
        exit_code = set_exit_code(
            int(parsed_data['instant_cell_usage']), exit_code
            )
 
   
    # Now get per-port stats.  We add together each of the
    # 8 buffer queues for simplicity here, if that doesn't
    # suit your purposes please feel free to modify.
    for port_num in args.interfaces:
        if int(port_num) in parsed_data:
            # Pickle that port data.
            pickle_data.append(
                tuple(
                    ['iface_instant_cell_usage.1-%s.%s' % (port_num, hostname),
                tuple([get_cmd_timestamp, int(parsed_data[int(port_num)])])])
                )
            logger.debug("Pickled instant cell usage for 1/%s: %s" % (
                port_num, parsed_data[int(port_num)]
                ))
 
   
            # We're also doing a metric for the percentage of
            # alpha threshhold used.  In a nutshell, a packet
            # is only admitted to the buffer if an threshhold is
            # not exceeded.  The threshhold is the remaining
            # instant usage (taken from the <rem_instant_usage> tag
            # in the output we parsed) times 2.  Because this threhshold
            # is dependent on how much buffer is actually in use at
            # any given time, we graph the current buffer utilization
            # on the port as a percentage of the threshhold.  When
            # we hit 100%, no more packets will be admitted to the buffer
            # on this port even if there is buffer available on the box.
            percent_used = float(parsed_data[int(port_num)]) / (
                int(parsed_data['rem_instant_usage']) * 2) * 100
            pickle_data.append(
                tuple([
                    'percent_buf_threshold.1-%s.%s' % (port_num, hostname),
                    tuple([get_cmd_timestamp, percent_used])
                    ]))
            logger.debug("Pickled percent of threshhold for 1/%s: %f" % (
                port_num, percent_used
                ))
        else:
            print """
    WARNING: requested interface %s not found in command output.
    """ % (port_num)
 
   
 
   
def do_switch_commands():
    """
    A hook function for executing any switch-level command necessary.
    Commands for individual interfaces are handled elsewhere.
    """
    global args
    # TODO (mvoelker): add CLI options here to determine which
    # commands get run.
    if args.get_queuing_stats:
        get_show_queuing_int()
    if args.get_buffer_stats:
        get_buffer_stats()
 
   
 
   
def do_interface_commands():
    """
    A hook function for executing any per-interface command necessary.
    Commands for handling switch-level stats and commands which
    provide data for multiple interfaces are generally handled in
    do_switch_commands().
    """
    global args
    if args.get_int_counters:
        get_int_counters()
 
   
# Provide usage and parse command line options.
usage = "\n%prog [options] [arg1 arg2 ...]"
usage += """
 
   
Arguments are the numbers of the ports you want to collect
buffer queue stats for.  If unspecified, no per-port stats
will be displayed.
 
   
This script is intended to be run on a Cisco Nexus 3000-series switch
(tested on 3064 and 3048 models).  It can be run manually or via
the NX-OS scheduler.  It will report buffer utilization stats
parsed from the output of "show hardware internal buffer pkt-stats detail"
via the pickle protocol over TCP to Graphite (or another data
sink of your choice that can grock pickled data).
 
   
Example:
%prog -H myN3K.mydomain.com -l admin -p password \\
   -m -i 46 47 48
"""
parser = argparse.ArgumentParser(description=usage)
parser.add_argument("-H", "--hostname", dest="hostname",
    help="Hostname or IP address", required=True)
parser.add_argument("-p", "--pidfile", dest="pidfile",
    help="File in which to write our PID", default="/bootflash/buffer_check.py.pid")
parser.add_argument("-v", "--verbose", dest="verbosity", action="count",
    help="Enable verbose output.", default=0)
parser.add_argument("-b", "--clear_buffer_counters", dest="clear_counters",
    help="Clear buffer counters after checking", default=False,
    action="store_true")
parser.add_argument("-m", "--max_buffer", dest="get_max_buf",
    help="Show max buffer utilization", default=False,
    action="store_true")
parser.add_argument("-i", "--instant_buffer", dest="get_instant_buf",
    help="Show instant buffer utilization", default=False,
    action="store_true")
parser.add_argument("interfaces", metavar="N", type=int, nargs='*',
    help='List of interfaces to check.')
parser.add_argument("-s", "--sleep_interval", dest="sleep_interval",
    help="Interval to sleep between polls (higher reduces CPU hit)",
    type=float, default=0)
parser.add_argument("-q", "--queuing_stats", dest="get_queuing_stats",
    help="Get stats from 'show queuing interface'", default=False,
    action="store_true")
parser.add_argument("-c", "--interface_counters", dest="get_int_counters",
    help="Get stats from 'show interface x/x'", default=False,
    action="store_true")
parser.add_argument("-f", "--buffer_stats", dest="get_buffer_stats",
    help="Get stats from 'show hardware internal buffer pkts-stats detail'",
    default=False, action="store_true")
args = parser.parse_args()
 
   
# Set up a logger.
logger = logging.getLogger('n3k_buffer_check')
logging.basicConfig()
 
   
# Since this started out purely as a script for buffer monitoring commands,
# certain command options imply others.  Fix things up here.
if args.get_instant_buf:
    args.get_buffer_stats = True
    logger.debug("CLI: assuming -f because I received -i.")
if args.get_max_buf:
    args.get_buffer_stats = True
    logger.debug("CLI: assuming -f because I received -m.")
if args.clear_counters:
    args.get_buffer_stats = True
    logger.debug("CLI: assuming -f because I received -b.")
 
   
# Daemonize ourself if we've gotten this far.
daemonize()
 
   
# Set the hostname.
hostname = socket.gethostname()
 
   
# If we're doing verbose output, set that up.
if args.verbosity == 3:
    # Hmm...do....something?
    logger.setLevel(logging.DEBUG)
if args.verbosity >= 2:
    # Hmm...do...something else?
    logger.setLevel(logging.DEBUG)
if args.verbosity >= 1:
    logger.setLevel(logging.DEBUG)
if args.verbosity == 0:
    # Redirect standard I/O streams to /dev/null.
    os.close(0)
    os.close(1)
    os.close(2)
    os.open(os.devnull, os.O_RDWR)
    os.dup2(0, 1)
    os.dup2(0, 2)
 
   
# Add a message for clearing the counters.
clear_message = "clear counters buffers"
 
   
# Set up some data holders to be used by XML parsing callback routines.
parsed_data = dict()
interface_rates = dict()
current_int = 0
current_tag = ''
charbuff = ''
port_num = 0
get_cmd_timestamp = 0.0
 
   
# A place to hold data we'll send back over the wire.
pickle_data = list()
 
   
# Set up a default exit code.
exit_code = 0
 
   
# Start up a socket over which to send data.
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((args.hostname, 2004))
 
   
while True:
    # Clear out old pickled data.
    pickle_data = list()
 
   
    # If we have other interface-level commands to do, do them
    # here.
    logger.debug("Doing interface-level commands...")
    do_interface_commands()
 
   
    # If we have other switch-level commands to do, do them here.
    logger.debug("Doing switch-level commands...")
    do_switch_commands()
 
   
    if args.verbosity > 0:
        logger.debug(pickle_data)
 
   
    # Pickle the data.
    payload = cPickle.dumps(pickle_data)
    logger.debug("Size of picked data: %s" % sys.getsizeof(payload))
    header = struct.pack("!L", len(payload))
    message = header + payload
 
   
    # Batch the data off to Graphite.
    # Unfortunately Carbon doesn't listen for pickle data on UDP
    # sockets. =(  If we can fix that, uncomment the next two lines
    # and comment out the three after that to use UDP instead of TCP.
    #sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    #sock.sendto(message, (args.hostname, 2004))
    sock.sendall(message)
 
   
    # Go to sleep if we've been told to do so.
    time.sleep(args.sleep_interval)

check_process.py

#!/usr/bin/python
import os
import sys
import re
import argparse
from cisco import CLI
 
   
usage = "\n%prog [options]"
usage += """
 
   
This script checks to see if the buffer_check.py script
is running by reading it's pidfile and pinging the listed pid.
If buffer_check.py isn't running, this script will start it.
 
   
If buffer_check.py is running, this script can optionally
kill it if run with the -k option.
 
   
Example:
%prog -k
"""
 
   
parser = argparse.ArgumentParser(description=usage)
parser.add_argument("-k", "--kill", dest="kill",
    help="Kill buffer_check.py if running", default=False,
    action="store_true")
args = parser.parse_args()
 
   
 
   
def start_process():
    """
    Starts the buffer_check.py script.  For our implementation, we
    start three instances: one to check buffer stats, one to check
    interface stats on server-facing ports, and one to check interface
    stats on other ports and queuing stats (at lower granularity).
    """
 
   
    cmd = 'python bootflash:buffer_check.py -H 172.18.117.181 -m -i -b -s 0.8 ' \
        '-f -p /bootflash/buffer_check.py.pid 1 2 3 4 5 6 7 8 17 18 19 20 21 ' \
        '33 34 35 36 37 46 47 48 57 58 59 60 61 62 63 64 '
    cli_obj = CLI(cmd, False)
    print "Started %s" % (cmd)
 
   
    cmd = 'python bootflash:buffer_check.py -H 172.18.117.181 -c -s 0.8 ' \
        '-p /bootflash/buffer_check.py-1.pid 33 34 35 36 37'
    cli_obj = CLI(cmd, False)
    print "Started %s" % (cmd)
 
   
    cmd = 'python bootflash:buffer_check.py -H 172.18.117.181 -c -q -s 5 ' \
        '-p /bootflash/buffer_check.py-2.pid 1 2 3 4 5 6 7 8 17 18 19 20 21' \
        '33 34 35 36 37 46 47 48 57 58 59 60 61 62 63 64 '
    cli_obj = CLI(cmd, False)
    print "Started %s" % (cmd)
 
   
 
   
def check_pid():
    """
    Checks to see if the buffer_check.py script is running.
    """
 
   
    pidfiles = ['/bootflash/buffer_check.py.pid', '/bootflash/buffer_check.py-1.pid',
                '/bootflash/buffer_check.py-2.pid']
 
   
    retval = True
 
   
    for pf in pidfiles:
        # Try to open our pidfile.
        try:
            f = open(pf, 'r')
        except IOError:
            print "No pidfile %s found!" % (pf)
            retval = False
 
   
        # Read the pid from the file and grock it down to an int.
        pid = f.readline()
        pidmatch = re.search('^(\d+)\s*$', pid)
        if pidmatch:
            pid = pidmatch.group(1)
            print "Pid from pidfile is %s" % (pid)
            global options
            try:
                if args.kill:
                    os.kill(int(pid), 9)
                    print "Killed %s" % (pid)
                else:
#!/usr/bin/python
import os
import sys
import re
import argparse
from cisco import CLI
 
   
usage = "\n%prog [options]"
usage += """
 
   
This script checks to see if the buffer_check.py script
is running by reading it's pidfile and pinging the listed pid.
If buffer_check.py isn't running, this script will start it.
 
   
If buffer_check.py is running, this script can optionally
kill it if run with the -k option.
 
   
Example:
%prog -k
"""
 
   
parser = argparse.ArgumentParser(description=usage)
parser.add_argument("-k", "--kill", dest="kill",
    help="Kill buffer_check.py if running", default=False,
    action="store_true")
args = parser.parse_args()
 
   
 
   
def start_process():
    """
    Starts the buffer_check.py script.  For our implementation, we
    start three instances: one to check buffer stats, one to check
    interface stats on server-facing ports, and one to check interface
    stats on other ports and queuing stats (at lower granularity).
    """
 
   
    cmd = 'python bootflash:buffer_check.py -H 172.18.117.181 -m -i -b -s 0.8 ' \
        '-f -p /bootflash/buffer_check.py.pid 1 2 3 4 5 6 7 8 17 18 19 20 21 ' \
        '33 34 35 36 37 46 47 48 57 58 59 60 61 62 63 64 '
    cli_obj = CLI(cmd, False)
    print "Started %s" % (cmd)
 
   
    cmd = 'python bootflash:buffer_check.py -H 172.18.117.181 -c -s 0.8 ' \
        '-p /bootflash/buffer_check.py-1.pid 33 34 35 36 37'
    cli_obj = CLI(cmd, False)
    print "Started %s" % (cmd)
 
   
    cmd = 'python bootflash:buffer_check.py -H 172.18.117.181 -c -q -s 5 ' \
        '-p /bootflash/buffer_check.py-2.pid 1 2 3 4 5 6 7 8 17 18 19 20 21' \
        '33 34 35 36 37 46 47 48 57 58 59 60 61 62 63 64 '
    cli_obj = CLI(cmd, False)
    print "Started %s" % (cmd)
 
   
 
   
def check_pid():
    """
    Checks to see if the buffer_check.py script is running.
    """
 
   
    pidfiles = ['/bootflash/buffer_check.py.pid', '/bootflash/buffer_check.py-1.pid',
                '/bootflash/buffer_check.py-2.pid']
 
   
    retval = True
 
   
    for pf in pidfiles:
        # Try to open our pidfile.
        try:
            f = open(pf, 'r')
        except IOError:
            print "No pidfile %s found!" % (pf)
            retval = False
 
   
        # Read the pid from the file and grock it down to an int.
        pid = f.readline()
        pidmatch = re.search('^(\d+)\s*$', pid)
        if pidmatch:
            pid = pidmatch.group(1)
            print "Pid from pidfile is %s" % (pid)
            global options
            try:
                if args.kill:
                    os.kill(int(pid), 9)
                    print "Killed %s" % (pid)
                else:
                    os.kill(int(pid), 0)
            except OSError:
                print "%s is dead." % (pid)
                retval = False
            else:
                if not args.kill:
                    print "%s is alive." % (pid)
        else:
            print "No pid found!"
            retval = False
 
   
    return retval
 
   
if check_pid():
    # We can exit, the scripts are running.
    exit(0)
else:
    # We need to start the scripts.
    if not args.kill:
        start_process()
    exit(1)

NX-OS Scheduler Example

milliways-3k-1# show scheduler config 
config terminal
  feature scheduler
  scheduler logfile size 16
end
 
   
 
   
config terminal
 scheduler job name buffer_check
python bootflash:/check_process.py
 show scheduler config 
 
   
end
 
   
config terminal
  scheduler schedule name every_minute
    time start 2012:09:10:09:58 repeat 1

Collectd Configuration

The following collectd configurations are available for consideration:

collectd.conf

Puppet Manifest

collectd.conf

BaseDir     "/var/lib/collectd"
PIDFile     "/var/run/collectd.pid"
PluginDir   "/usr/lib64/collectd"
TypesDB     "/usr/share/collectd/types.db"
Interval     1
ReadThreads  5
LoadPlugin syslog
LoadPlugin cpu
LoadPlugin disk
LoadPlugin ethstat
LoadPlugin libvirt
LoadPlugin load
LoadPlugin memory
LoadPlugin write_graphite
<Plugin disk>
	Disk "/^[hs]d[a-f][0-9]?$/"
	IgnoreSelected false
</Plugin>
 
   
<Plugin ethstat>
	Interface "eth0"
        Interface "eth1"
        Map "rx_packets" "pkt_counters" "rx_packets"
        Map "tx_packets" "pkt_counters" "tx_packets"
        Map "rx_bytes" "byte_counters" "rx_bytes"
        Map "tx_bytes" "byte_counters" "tx_bytes"
        Map "rx_errors" "error_counters" "rx_errors"
        Map "tx_errors" "error_counters" "tx_errors"
        Map "rx_dropped" "drop_counters" "rx_dropped"
        Map "tx_dropped" "drop_counters" "tx_dropped"
        Map "collisions" "error_counters" "collisions"
        Map "rx_over_errors" "error_counters" "rx_over_errors"
        Map "rx_crc_errors" "error_counters" "rx_crc_errors"
        Map "rx_frame_errors" "error_counters" "rx_frame_errors"
        Map "rx_fifo_errors" "error_counters" "rx_fifo_errors"
        Map "rx_missed_errors" "error_counters" "rx_missed_errors"
        Map "tx_aborted_errors" "error_counters" "tx_aborted_errors"
        Map "tx_carrier_errors" "error_counters" "tx_carrier_errors"
        Map "tx_fifo_errors" "error_counters" "tx_fifo_errors"
        Map "tx_heartbeat_errors" "error_counters" "tx_heartbeat_errors"
        Map "rx_pkts_nic" "pkt_counters" "rx_pkts_nic"
        Map "tx_pkts_nic" "pkt_counters" "tx_pkts_nic"
        Map "rx_bytes_nic" "byte_counters" "rx_bytes_nic"
        Map "tx_bytes_nic" "byte_counters" "tx_bytes_nic"
        Map "lsc_int" "misc_counters" "lsc_int"
        Map "tx_busy" "error_counters" "tx_busy"
        Map "non_eop_descs" "misc_counters" "non_eop_descs"
        Map "broadcast" "pkt_counters" "broadcast"
        Map "rx_no_buffer_count" "error_counters" "rx_no_buffer_count"
        Map "tx_timeout_count" "error_counters" "tx_timeout_count"
        Map "tx_restart_queue" "error_counters" "tx_restart_queue"
        Map "rx_long_length_errors" "error_counters" "rx_long_length_errors"
        Map "rx_short_length_errors" "error_counters" "rx_short_length_errors"
        Map "tx_flow_control_xon" "misc_counters" "tx_flow_control_xon"
        Map "rx_flow_control_xon" "misc_counters" "rx_flow_control_xon"
        Map "tx_flow_control_xoff" "misc_counters" "tx_flow_control_xoff"
        Map "rx_flow_control_xoff" "misc_counters" "rx_flow_control_xoff"
        Map "rx_csum_offload_errors" "error_counters" "rx_csum_offload_errors"
        Map "alloc_rx_page_failed" "error_counters" "alloc_rx_page_failed"
        Map "alloc_rx_buff_failed" "error_counters" "alloc_rx_buff_failed"
        Map "rx_no_dma_resources" "error_counters" "rx_no_dma_resources"
        Map "hw_rsc_aggregated" "misc_counters" "hw_rsc_aggregated"
        Map "hw_rsc_flushed" "misc_counters" "hw_rsc_flushed"
	MappedOnly true
</Plugin>
 
   
<Plugin libvirt>
	Connection "qemu:///system"
	RefreshInterval 5
	IgnoreSelected false
          HostnameFormat hostname name
</Plugin>
 
   
<Plugin write_graphite>
  <Carbon>
    Host "voyager-graphite.hosts.voyager.cisco.com"
    Port "2003"
    Prefix "collectd"
    Postfix "collectd"
    StoreRates false
    AlwaysAppendDS false
    EscapeCharacter "_"
  </Carbon>
</Plugin>
 
   
Include "/etc/collectd.d"

Puppet Manifest

class collectd {
 
   
    package { "collectd":
       name     => "collectd",
       ensure   => 'latest',
       require  => [File['/etc/yum.conf']]
    }
 
   
    package { "collectd-graphite":
       name     => "collectd-graphite",
       ensure   => 'latest',
       require  => [File['/etc/yum.conf']]
    }
 
   
    package { "collectd-ethstat":
       name     => "collectd-ethstat",
       ensure   => 'latest',
       require  => [File['/etc/yum.conf']]
    }
 
   
    package { "collectd-libvirt":
       name     => "collectd-libvirt",
       ensure   => 'latest',
       require  => [File['/etc/yum.conf']]
    }
 
   
    service { "collectd":
        enable  => 'true',
        ensure  => 'running',
        start   => '/etc/init.d/collectd start',
        stop    => '/etc/init.d/collectd stop',
        require => [Package['collectd'], Package['collectd-graphite'], 
Package['collectd-ethstat'], File['/etc/collectd.conf']]
    }
 
   
    if $fqdn =~ /^r05+-p0[1-5]\.hosts\.voyager\.cisco\.com$/ {
        file { '/etc/collectd.conf':
            #source  => 'puppet:///modules/collectd/collectd.conf.enabled',
            source  => 'puppet:///modules/collectd/collectd.conf',
            owner   => 'root',
            group   => 'root',
            mode    => '644',
            notify  => Service['collectd'],
            require => Package['collectd']
        }
    } else {
        file { '/etc/collectd.conf':
            source  => 'puppet:///modules/collectd/collectd.conf',
            owner   => 'root',
            group   => 'root',
            mode    => '644',
            notify  => Service['collectd'],
            require => Package['collectd']
        }
    }
}

Graphite Configuration

The following graphite configurations are available for consideration:

carbon.conf

graphite.wsgi

graphite-vhost.conf

local_settings.py

relay-rules.conf

storage-schemas.conf

Puppet Manifest (init.pp)

carbon.conf

[cache]
USER =
MAX_CACHE_SIZE = inf
MAX_UPDATES_PER_SECOND = 50000
MAX_CREATES_PER_MINUTE = 500
LINE_RECEIVER_INTERFACE = 0.0.0.0
LINE_RECEIVER_PORT = 2103
ENABLE_UDP_LISTENER = True
UDP_RECEIVER_INTERFACE = 0.0.0.0
UDP_RECEIVER_PORT = 2103
PICKLE_RECEIVER_INTERFACE = 0.0.0.0
PICKLE_RECEIVER_PORT = 2104
USE_INSECURE_UNPICKLER = False
CACHE_QUERY_INTERFACE = 0.0.0.0
CACHE_QUERY_PORT = 7102
USE_FLOW_CONTROL = True
LOG_UPDATES = False
WHISPER_AUTOFLUSH = False
 
   
[cache:b]
LINE_RECEIVER_PORT = 2203
PICKLE_RECEIVER_PORT = 2204
CACHE_QUERY_PORT = 7202
UDP_RECEIVER_PORT = 2203
 
   
[cache:c]
LINE_RECEIVER_PORT = 2303
PICKLE_RECEIVER_PORT = 2304
CACHE_QUERY_PORT = 7302
UDP_RECEIVER_PORT = 2303
 
   
[cache:d]
LINE_RECEIVER_PORT = 2403
PICKLE_RECEIVER_PORT = 2404
CACHE_QUERY_PORT = 7402
UDP_RECEIVER_PORT = 2403
 
   
[cache:e]
LINE_RECEIVER_PORT = 2503
PICKLE_RECEIVER_PORT = 2504
CACHE_QUERY_PORT = 7502
UDP_RECEIVER_PORT = 2503
 
   
[cache:f]
LINE_RECEIVER_PORT = 2603
PICKLE_RECEIVER_PORT = 2604
CACHE_QUERY_PORT = 7602
UDP_RECEIVER_PORT = 2603
 
   
[cache:g]
LINE_RECEIVER_PORT = 2703
PICKLE_RECEIVER_PORT = 2704
CACHE_QUERY_PORT = 7702
UDP_RECEIVER_PORT = 2703
 
   
[cache:h]
LINE_RECEIVER_PORT = 2803
PICKLE_RECEIVER_PORT = 2804
CACHE_QUERY_PORT = 7802
UDP_RECEIVER_PORT = 2803
 
   
[cache:i]
LINE_RECEIVER_PORT = 2903
PICKLE_RECEIVER_PORT = 2904
CACHE_QUERY_PORT = 7902
UDP_RECEIVER_PORT = 2903
 
   
[cache:j]
LINE_RECEIVER_PORT = 3003
PICKLE_RECEIVER_PORT = 3004
CACHE_QUERY_PORT = 8002
UDP_RECEIVER_PORT = 3003
 
   
[relay]
LINE_RECEIVER_INTERFACE = 0.0.0.0
LINE_RECEIVER_PORT = 2003
PICKLE_RECEIVER_INTERFACE = 0.0.0.0
PICKLE_RECEIVER_PORT = 2004
RELAY_METHOD = rules
REPLICATION_FACTOR = 1
DESTINATIONS = 127.0.0.1:2104:a, 127.0.0.1:2204:b, 127.0.0.1:2304:c, 127.0.0.1:2404:d, 
127.0.0.1:2504:e, 127.0.0.1:2604:f, 127.0.0.1:2704:g, 127.0.0.1:2804:h, 
127.0.0.1:2904:i, 127.0.0.1:3004:j
MAX_DATAPOINTS_PER_MESSAGE = 500
MAX_QUEUE_SIZE = 10000
USE_FLOW_CONTROL = True
 
   
[aggregator]
LINE_RECEIVER_INTERFACE = 0.0.0.0
LINE_RECEIVER_PORT = 2023
PICKLE_RECEIVER_INTERFACE = 0.0.0.0
PICKLE_RECEIVER_PORT = 2024
DESTINATIONS = 127.0.0.1:2004
REPLICATION_FACTOR = 1
MAX_QUEUE_SIZE = 10000
USE_FLOW_CONTROL = True
MAX_DATAPOINTS_PER_MESSAGE = 500
MAX_AGGREGATION_INTERVALS = 5

graphite.wsgi

import os, sys
sys.path.append('/opt/graphite/webapp')
os.environ['DJANGO_SETTINGS_MODULE'] = 'graphite.settings'
import django.core.handlers.wsgi
application = django.core.handlers.wsgi.WSGIHandler()
from graphite.logger import log
log.info("graphite.wsgi - pid %d - reloading search index" % os.getpid())
import graphite.metrics.search

graphite-vhost.conf

<IfModule !wsgi_module.c>
    LoadModule wsgi_module modules/mod_wsgi.so
</IfModule>
 
   
WSGISocketPrefix run/wsgi
 
   
<VirtualHost *:80>
        ServerName voyager-graphite
        ServerAlias voyager-graphite.cisco.com
        DocumentRoot "/opt/graphite/webapp"
        ErrorLog /opt/graphite/storage/log/webapp/error.log
        CustomLog /opt/graphite/storage/log/webapp/access.log common
 
   
        WSGIDaemonProcess graphite processes=16 threads=16 display-name='%{GROUP}' 
inactivity-timeout=120
        WSGIProcessGroup graphite
        WSGIApplicationGroup %{GLOBAL}
        WSGIImportScript /opt/graphite/conf/graphite.wsgi process-group=graphite 
application-group=%{GLOBAL}
        WSGIScriptAlias / /opt/graphite/conf/graphite.wsgi 
 
   
        Alias /content/ /opt/graphite/webapp/content/
        <Location "/content/">
                SetHandler None
        </Location>
 
   
        Alias /media/ "@DJANGO_ROOT@/contrib/admin/media/"
        <Location "/media/">
                SetHandler None
        </Location>
 
   
        <Directory /opt/graphite/conf/>
                Order deny,allow
                Allow from all
        </Directory>
 
   
</VirtualHost>

local_settings.py

TIME_ZONE = 'America/New_York'
DEBUG = True
USE_LDAP_AUTH = True
LDAP_SERVER = "ldap.cisco.com"
LDAP_PORT = 389
LDAP_SEARCH_BASE = "OU=active,OU=employees,ou=people,o=cisco.com"
LDAP_USER_QUERY = "(uid=%s)"  #For Active Directory use "(sAMAccountName=%s)"
CARBONLINK_HOSTS = ["127.0.0.1:7102:a", "127.0.0.1:7202:b", "127.0.0.1:7302:c", 
"127.0.0.1:7402:d", "127.0.0.1:7502:e", "127.0.0.1:7602:f", "127.0.0.1:7702:g", 
"127.0.0.1:7802:h", "127.0.0.1:7902:i", "127.0.0.1:8002:j"]

relay-rules.conf

[collectd01-02]
pattern = collectdr01.*
destinations = 127.0.0.1:2104:a
 
   
[collectd03-04]
pattern = collectdr03.*
destinations = 127.0.0.1:2204:b
 
   
[collectd05-06]
pattern = collectdr05.*
destinations = 127.0.0.1:2304:c
 
   
[collectd07-08]
pattern = collectdr07.*
destinations = 127.0.0.1:2404:d
 
   
[collectd09-10]
pattern = collectdr09.*
destinations = 127.0.0.1:2504:e
 
   
[collectd11-12]
pattern = collectdr11.*
destinations = 127.0.0.1:2604:f
 
   
[collectd13-14]
pattern = collectdr13.*
destinations = 127.0.0.1:2704:g
 
   
[collectd15-16]
pattern = collectdr15.*
destinations = 127.0.0.1:2804:h
 
   
[iface_eth_inb]
pattern = iface_eth_inb.*
destinations = 127.0.0.1:2904:i
 
   
[iface_eth_inp]
pattern = iface_eth_inp.*
destinations = 127.0.0.1:2904:i
 
   
[iface_eth_outb]
pattern = iface_eth_outb.*
destinations = 127.0.0.1:3004:j
 
   
[iface_eth_outp]
pattern = iface_eth_outp.*
destinations = 127.0.0.1:3004:j
 
   
[max_cell]
pattern = max_cell.*
destinations = 127.0.0.1:2504:e
 
   
[instant_cell]
pattern = instant_cell.*
destinations = 127.0.0.1:2604:f
 
   
[percent_buf]
pattern = percent_buf.*
destinations = 127.0.0.1:2704:g
 
   
[carbon]
pattern = carbon.*
destinations = 127.0.0.1:3004:j
 
   
[default]
default = true
destinations = 127.0.0.1:2904:i

storage-schemas.conf

[carbon]
pattern = ^carbon\.
retentions = 60:90d
 
   
[interface_max_buffer]
pattern = ^max_cell_usage* 
retentions = 1s:10d,1m:30d
 
   
[interface_instant_buffer]
pattern = ^instant_cell_usage* 
retentions = 1s:10d,1m:30d
 
   
[interface_percent_threshhold]
pattern = ^iface_instant_cell_usage*
retentions = 1s:10d,1m:30d
 
   
[collectd]
pattern = ^collectd*
retentions = 1s:10d,1m:30d
 
   
[selective_in_byte_count]
pattern = ^iface_eth_inbytes+?\.1-3\d*
retentions = 1s:10d,1m:30d
 
   
[selective_out_byte_count]
pattern = ^iface_eth_outbytes+?\.1-3\d*
retentions = 1s:10d,1m:30d
 
   
[selective_in_bit_count]
pattern = ^iface_eth_inbits_rate\.1-3\d*
retentions = 1s:10d,1m:30d
 
   
[selective_out_bit_count]
pattern = ^iface_eth_outbits_rate\.1-3\d*
retentions = 1s:10d,1m:30d
 
   
[default_1min_for_1day]
pattern = .*
retentions = 10s:10d,1m:30d

Puppet Manifest (init.pp)

class graphite {
    package { "mod_wsgi":
       name     => "mod_wsgi",
       ensure   => "installed"
    }
 
   
    package { "gcc":
       name     => "gcc",
       ensure   => "installed",
    }
 
   
    package { "pycairo":
       name     => "pycairo",
       ensure   => 'installed',
    }
 
   
    package { "mod_python":
       name     => "mod_python",
       ensure   => 'installed',
    }
 
   
    package { "Django":
       name     => "Django",
       ensure   => 'installed',
    }
    
    package { "django-tagging":
       name     => "django-tagging",
       ensure   => 'installed',
    }
 
   
    package { "python-ldap":
       name     => "python-ldap",
       ensure   => 'installed',
    }
 
   
    package { "python-memcached":
       name     => "python-memcached",
       ensure   => 'installed',
    }
 
   
    package { "python-sqlite2":
       name     => "python-sqlite2",
       ensure   => 'installed',
    }
 
   
    package { "bitmap":
       name     => "bitmap",
       ensure   => 'installed',
    }
 
   
    package { "bitmap-fixed-fonts":
       name     => "bitmap-fixed-fonts",
       ensure   => 'installed',
    }
 
   
    package { "bitmap-fonts-compat":
       name     => "bitmap-fonts-compat",
       ensure   => 'installed',
    }
 
   
    package { "python-devel":
       name     => "python-devel",
       ensure   => 'installed',
    }
 
   
    package { "python-crypto":
       name     => "python-crypto",
       ensure   => 'installed',
    }
 
   
    package { "pyOpenSSL":
       name     => "pyOpenSSL",
       ensure   => 'installed',
    }
 
   
    package { "graphite-web":
       name     => "graphite-web",
       ensure   => 'installed',
       provider => 'pip',
       require  => [Package['pycairo'], Package['mod_python'], Package['Django'], 
Package['python-ldap'], Package['python-memcached'], Package['python-sqlite2'], 
Package['bitmap'], Package['bitmap-fonts-compat'], Package['bitmap-fixed-fonts']]
    }
 
   
    package { "carbon":
       name     => "carbon",
       ensure   => 'installed',
       provider => 'pip',
       require  => [Package['pycairo'], Package['mod_python'], Package['Django'], 
Package['python-ldap'], Package['python-memcached'], Package['python-sqlite2'], 
Package['bitmap'], Package['bitmap-fonts-compat'], Package['bitmap-fixed-fonts']]
    }
 
   
    package { "whisper":
       name     => "whisper",
       ensure   => 'installed',
       provider => 'pip',
       require  => [Package['pycairo'], Package['mod_python'], Package['Django'], 
Package['python-ldap'], Package['python-memcached'], Package['python-sqlite2'], 
Package['bitmap'], Package['bitmap-fonts-compat'], Package['bitmap-fixed-fonts']]
    }
 
   
    file { '/opt/graphite/conf/carbon.conf':
        source  => 'puppet:///modules/graphite/carbon.conf',
        owner   => 'apache',
        group   => 'root',
        mode    => '644',
        require => Package['carbon']
    }
 
   
    file { '/opt/graphite/conf/storage-schemas.conf':
        source  => 'puppet:///modules/graphite/storage-schemas.conf',
        owner   => 'apache',
        group   => 'root',
        mode    => '644',
        require => Package['whisper']
    }
 
   
    file { '/opt/graphite/conf/graphite.wsgi':
        source  => 'puppet:///modules/graphite/graphite.wsgi',
        owner   => 'apache',
        group   => 'root',
        mode    => '655',
        require => Package['graphite-web']
    }
 
   
    file { '/opt/graphite/webapp/local_settings.py':
        source  => 'puppet:///modules/graphite/local_settings.py',
        owner   => 'apache',
        group   => 'root',
        mode    => '655',
        require => Package['graphite-web']
    }
 
   
    file { '/etc/httpd/conf.d/graphite-vhost.conf':
        source  => 'puppet:///modules/graphite/graphite-vhost.conf',
        owner   => 'root',
        group   => 'root',
        mode    => '655',
        require => Package['graphite-web'],
        notify  => Service['httpd']
    }
 
   
    service { "httpd":
        enable  => 'true',
        ensure  => 'running',
        start   => '/etc/init.d/httpd start',
        stop    => '/etc/init.d/httpd stop',
        require => [Package['graphite-web'], 
File['/etc/httpd/conf.d/graphite-vhost.conf']]
    }
}