From: t.moriyama Date: Mon, 8 Apr 2013 12:34:22 +0000 (+0900) Subject: move collector, collectagents to collect package. X-Git-Url: http://git.osdn.net/view?p=ti2%2Fti2.git;a=commitdiff_plain;h=cd18d4e65226c77cedc29246ed651617fabf6c49 move collector, collectagents to collect package. --- diff --git a/linkpair/collect/__init__.py b/linkpair/collect/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/linkpair/collect/__init__.pyc b/linkpair/collect/__init__.pyc new file mode 100644 index 0000000..9fdd6b9 Binary files /dev/null and b/linkpair/collect/__init__.pyc differ diff --git a/linkpair/collect/agent/__init__.py b/linkpair/collect/agent/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/linkpair/collect/agent/__init__.pyc b/linkpair/collect/agent/__init__.pyc new file mode 100644 index 0000000..f0000e2 Binary files /dev/null and b/linkpair/collect/agent/__init__.pyc differ diff --git a/linkpair/collect/agent/commandrunner.py b/linkpair/collect/agent/commandrunner.py new file mode 100755 index 0000000..2a76309 --- /dev/null +++ b/linkpair/collect/agent/commandrunner.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# + +__version__ = '1.1' + +import sys +import os +import re +from subprocess import Popen, PIPE +from socket import gethostname +import shlex +# import paramiko as ssh +import ssh +from linkpair.commonutils import CommonUtils + + +class CommandRunner(object): + ''' + Command runner + ''' + + PEER_FOUND = 1 + + def __init__(self, remote_desc): + self._u = CommonUtils() + self._remote_desc = remote_desc + self._ssh_username = "" + self._ssh_hostname = "" + self._ssh_hostport = 22 + self._remote_password = "" + self._remote_sshkey = "" + self._ssh_keyauth = False + self._ssh_passauth = False + + ''' set parameters ''' + [self._ssh_username, self._ssh_hostname, self._ssh_hostport] = self._u.parse_remote_desc(remote_desc) + + def exec_cmd(self, cmdline): + args = shlex.split(cmdline) + if self._remote_desc is not None: + result = self._exec_cmd_on_ssh(cmdline) + else: + result = Popen(args, stdout=PIPE, stderr=PIPE).stdout.readlines() + return result + + def _exec_cmd_on_ssh(self, cmdline): + sshc = ssh.SSHClient() + sshc.load_system_host_keys() + + if self._ssh_keyauth == True: + sshc.connect(self._ssh_hostname, username=self._ssh_username, + port=self._ssh_hostport, key_filename=self._remote_sshkey) + elif self._ssh_passauth == True: + sshc.connect(self._ssh_hostname, username=self._ssh_username, + port=self._ssh_hostport, password=self._remote_password) + else: + print "SSH connections failed" + sys.exit(1) + + stdin, stdout, stderr = sshc.exec_command(cmdline) + result = stdout.read().splitlines() + return result + + def set_remote_sshkey(self, remote_sshkey): + self._remote_sshkey = remote_sshkey + self._ssh_keyauth = True + + def set_remote_password(self, remote_password): + self._remote_password = remote_password + self._ssh_passauth = True + + \ No newline at end of file diff --git a/linkpair/collect/agent/commandrunner.pyc b/linkpair/collect/agent/commandrunner.pyc new file mode 100644 index 0000000..885ab3a Binary files /dev/null and b/linkpair/collect/agent/commandrunner.pyc differ diff --git a/linkpair/collect/agent/ovs.py b/linkpair/collect/agent/ovs.py new file mode 100755 index 0000000..604ce7c --- /dev/null +++ b/linkpair/collect/agent/ovs.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# + +__version__ = '1.1' + +import sys +import os +import re +from xml.dom import minidom +from subprocess import Popen, PIPE +from socket import gethostname +from device import Device +from port import Port +from linkpair import LinkPair +from formatter.grapheasy import GraphEasyFormatter +from commonutils import CommonUtils + + +class OvsAgent(object): + ''' + Open vSwitch Collector Agent + ''' + + PEER_FOUND = 1 + + def __init__(self, linkpairs, runner, formatter=GraphEasyFormatter()): + self._linkpairs = linkpairs + self._port_to_br = {} + self._iface_to_nss = {} + self._u = CommonUtils() + self._runner = runner + + ''' set parameters ''' + self._dbu = dbu + self._formatter = formatter + + def get_configuration(self): + patch_peers = {} + + result = self._exec_cmd("ovs-vsctl list-br") + for br_src in result: + br_src = br_src.rstrip() + result2 = self._exec_cmd("ovs-dpctl show " + br_src) + for port_desc in result2: + port_desc = port_desc.rstrip() + if self._u.d_push( + re.search(r'port \d+: (.*?) \(patch: peer=(.*?)\)', + port_desc)) is not None: + match = self._u.d_pop() + patch_src = match.group(1) + patch_dst = match.group(2) + patch_peers[patch_src + ":" + patch_dst] = self.PEER_FOUND + + if patch_dst + ":" + patch_src in patch_peers: + continue + + result3 = self._exec_cmd( + "ovs-vsctl port-to-br " + patch_dst) + if result3 is not None and len(result3) > 0: + br_dst = result3[0].rstrip() + self.add_linkpair( + Device(br_src, Device.BR_TYPE), + Device(br_dst, Device.BR_TYPE), + Port(patch_src), + Port(patch_dst), + self._formatter.PATCH_FORMAT) + else: + self.add_linkpair( + Device(br_src, Device.BR_TYPE), + Device("NOT CONNECTED", Device.BR_TYPE), + Port(patch_src), + Port(patch_dst), + self._formatter.PATCH_FORMAT) + + else: + # Internal OVSPort. + if self._u.d_push( + re.search( + r'port \d+: ' + br_src + ' \(internal\)', + port_desc)) is None: + if self._u.d_push(re.search(r'port \d+: (.*)', port_desc)) is not None: + port = self._u.d_pop().group(1).rstrip() + if self._u.d_push(re.match(r'^eth\d+$', port)) \ + or self._u.d_push(re.match(r'^em\d+$', port)) \ + or self._u.d_push(re.match(r'^igb\d+$', port)): + self._port_to_br[port] = br_src + self.add_linkpair( + Device(br_src, Device.BR_TYPE), + Device("Physical NW", Device.PHYNET_TYPE), + Port(port), + Port("")) + elif self._u.d_push(re.match(r'(vxlan\d+)', port)) \ + or self._u.d_push(re.match(r'(gre\d+)', port)) \ + or self._u.d_push(re.match(r'(gre-\d+)', port)): + port2 = self._u.d_pop().group(1) + self._port_to_br[port2] = br_src + self.add_linkpair( + Device(br_src, Device.BR_TYPE), + Device("OS Routing", Device.OS_ROUTE_TYPE), + Port(port), + Port("")) + elif re.search(r' \(internal\)', port): + port = re.sub(r' \(internal\)', '', port) + if port in self._iface_to_nss: + self.add_linkpair( + Device(br_src, Device.BR_TYPE), + Device(self._iface_to_nss[ + port], Device.NAMESPACE_TYPE), + Port(port), + Port(""), + self._formatter.NAMESPACE_FORMAT) + else: + self.add_linkpair( + Device(br_src, Device.BR_TYPE), + Device( + "INTERNAL", Device.OS_ROUTE_TYPE), + Port(port), + Port("")) + else: + ## Other OVSPort + self._port_to_br[port] = br_src + else: + continue diff --git a/linkpair/collect/collector.py b/linkpair/collect/collector.py new file mode 100755 index 0000000..1b6cec7 --- /dev/null +++ b/linkpair/collect/collector.py @@ -0,0 +1,288 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# + +__version__ = '1.1' + +import sys +import os +import re +from xml.dom import minidom +from subprocess import Popen, PIPE +from socket import gethostname +import shlex +import libvirt +from linkpair.device import Device +from linkpair.port import Port +from linkpair.linkpair import LinkPair +from linkpair.formatter.grapheasy import GraphEasyFormatter +from linkpair.commonutils import CommonUtils +from linkpair.dbutils import DBUtils +from linkpair.collect.agent.commandrunner import CommandRunner + +class Collector(object): + '''LinkpPair collector + + This class gets the LinkPair information from LinuxBridge and Open vSwitch and libvirt + ''' + + PEER_FOUND = 1 + + def __init__(self, remote_desc, dbu, formatter=GraphEasyFormatter()): + self._linkpairs = [] + self._port_to_br = {} + self._iface_to_nss = {} + self._u = CommonUtils() + self._db_enable = False + self._sql_conn = None + self._remote_desc = remote_desc + self._ssh_username = "" + self._ssh_hostname = "" + self._ssh_hostport = 22 + self._remote_password = "" + self._remote_sshkey = "" + self._ssh_keyauth = False + self._ssh_passauth = False + + ''' set parameters ''' + [self._ssh_username, self._ssh_hostname, self._ssh_hostport] = self._u.parse_remote_desc(remote_desc) + self._dbu = dbu + self._formatter = formatter + self._runner = CommandRunner(self._remote_desc) + + def run(self): + self.map_port_to_namespace() + self.pick_ovs_configuration() + self.pick_bridge_configuration() + self.pick_libvirt_configuration() + + def add_linkpair(self, dev1, dev2, port1, port2, format=""): + if format == "": + format = self._formatter.DEFAULT_FORMAT + self._linkpairs.append( + LinkPair(dev1, dev2, port1, port2, format)) +# if self._dbu.enable_db: +# insert_record(fmt, src, src_style, label, dst, dst_style) + + def get_linkpairs(self): + return self._linkpairs + + def drop_linkpairs(self): + self._linkpairs = [] + + def pick_libvirt_configuration(self): + virt_conn = self.get_libvirt_conn() + for id in virt_conn.listDomainsID(): + vm = virt_conn.lookupByID(id) + vmXMLDesc = minidom.parseString(vm.XMLDesc(0)) + for iface in vmXMLDesc.getElementsByTagName("interface"): + [device, bridge] = self.pick_libvirt_iface_configuration(iface) + mac = iface.getElementsByTagName( + "mac")[0].getAttribute("address") + device = iface.getElementsByTagName( + "target")[0].getAttribute("dev") + bridge = self.regist_to_port2br(device, bridge) + self.add_linkpair( + Device(str(vm.name()), Device.VM_TYPE), + Device(bridge, Device.BR_TYPE), + Port(device), + Port("")) + + def pick_libvirt_iface_configuration(self, iface): + ifaceType = iface.getAttribute("type") + bridge = "" + device = "" + if ifaceType == "network": + network = iface.getElementsByTagName( + "source")[0].getAttribute("network") + netXMLDesc = minidom.parseString( + virt_conn.networkLookupByName(network).XMLDesc(0)) + bridge = netXMLDesc.getElementsByTagName( + "bridge")[0].getAttribute("name") + elif ifaceType == "bridge": + bridge = iface.getElementsByTagName( + "source")[0].getAttribute("bridge") + return [device, bridge] + + def regist_to_port2br(self, device, bridge): + if device in self._port_to_br: + if bridge == "": + return self._port_to_br[device] + else: + return bridge + else: + self._port_to_br[device] = bridge + return bridge + + def map_port_to_namespace(self): + result = self._runner.exec_cmd("ip netns") + # if result.... + for ns in result: + ns = ns.rstrip() + result2 = self._runner.exec_cmd("ip netns exec " + ns + " ip link show") + for linkpair_out in result2: + linkpair_out = linkpair_out.rstrip() + match = re.match(r'\d+: (.*?): ', linkpair_out) + if match is not None and match.group(1) != 'lo': + self._iface_to_nss[match.group(1).rstrip()] = ns + + def pick_bridge_configuration(self): + br_name = "" + result = self._runner.exec_cmd("brctl show") + for br_line in result: + br_line = br_line.rstrip() + if self._u.d_push(re.match(r'^(\S+)\s+(\S+)\s+(\S+)\s+(\S+)$', br_line)) is not None: + match = self._u.d_pop() + br_name = match.group(1) + port = match.group(4) + if port not in self._port_to_br and br_name != "": + self._port_to_br[port] = br_name + ## for OpenStack quntum... + if self._u.d_push(re.match(r'^qvb(.+)', port)): + quantum_idprefix = self._u.d_pop().group(1) + if "qvo" + quantum_idprefix in self._port_to_br: + self.add_linkpair( + Device(br_name, Device.BR_TYPE), + Device(self._port_to_br[ + "qvo" + quantum_idprefix], Device.BR_TYPE), + Port(port), + Port("qvo" + quantum_idprefix), + self._formatter.VETH_FORMAT) + else: + self.add_linkpair( + Device(br_name), + Device("INTERNAL", Device.OS_ROUTE_TYPE), + Port(port), + Port("")) + # else: + # print self._port_to_br.keys() + # if self._u.d_push(re.match(r'^qvo(.+)', port)): + # continue + # add_linkpair(self.DEFAULT_FORMAT, br_name, self.DEFAULT_TYPE, port, \ + # "INTERNAL", self.OS_ROUTE_TYPE ) + + elif self._u.d_push(re.match(r'^\s+(\S+)$', br_line)) is not None: + port = self._u.d_pop().group(1) + if port not in self._port_to_br and br_name != "": + self._port_to_br[port] = br_name + ## for OpenStack quntum... + if self._u.d_push(re.match(r'^qvb(.+)', port)): + quantum_idprefix = self._u.d_pop().group(1) + if "qvo" + quantum_idprefix in self._port_to_br: + self.add_linkpair( + Device(br_name, Device.BR_TYPE), + Device(self._port_to_br[ + "qvo" + quantum_idprefix], Device.BR_TYPE), + Port(port), + Port("qvo" + quantum_idprefix), + self._formatter.VETH_FORMAT) + else: + if self._u.d_push(re.match(r'^qvo(.+)', port)): + continue + self.add_linkpair( + Device(br_name), + Device("INTERNAL", Device.OS_ROUTE_TYPE), + Port(port), + Port("")) + + def pick_ovs_configuration(self): + patch_peers = {} + + result = self._runner.exec_cmd("ovs-vsctl list-br") + for br_src in result: + br_src = br_src.rstrip() + result2 = self._runner.exec_cmd("ovs-dpctl show " + br_src) + for port_desc in result2: + port_desc = port_desc.rstrip() + if self._u.d_push( + re.search(r'port \d+: (.*?) \(patch: peer=(.*?)\)', + port_desc)) is not None: + match = self._u.d_pop() + patch_src = match.group(1) + patch_dst = match.group(2) + patch_peers[patch_src + ":" + patch_dst] = self.PEER_FOUND + + if patch_dst + ":" + patch_src in patch_peers: + continue + + result3 = self._runner.exec_cmd( + "ovs-vsctl port-to-br " + patch_dst) + if result3 is not None and len(result3) > 0: + br_dst = result3[0].rstrip() + self.add_linkpair( + Device(br_src, Device.BR_TYPE), + Device(br_dst, Device.BR_TYPE), + Port(patch_src), + Port(patch_dst), + self._formatter.PATCH_FORMAT) + else: + self.add_linkpair( + Device(br_src, Device.BR_TYPE), + Device("NOT CONNECTED", Device.NOT_CONNECTED_TYPE), + Port(patch_src), + Port(patch_dst), + self._formatter.PATCH_FORMAT) + + else: + # Internal OVSPort. + if self._u.d_push( + re.search( + r'port \d+: ' + br_src + ' \(internal\)', + port_desc)) is None: + if self._u.d_push(re.search(r'port \d+: (.*)', port_desc)) is not None: + port = self._u.d_pop().group(1).rstrip() + if self._u.d_push(re.match(r'^eth\d+$', port)) \ + or self._u.d_push(re.match(r'^em\d+$', port)) \ + or self._u.d_push(re.match(r'^igb\d+$', port)): + self._port_to_br[port] = br_src + self.add_linkpair( + Device(br_src, Device.BR_TYPE), + Device("Physical NW", Device.PHYNET_TYPE), + Port(port), + Port("")) + elif self._u.d_push(re.match(r'(vxlan\d+)', port)) \ + or self._u.d_push(re.match(r'(gre\d+)', port)) \ + or self._u.d_push(re.match(r'(gre-\d+)', port)): + port2 = self._u.d_pop().group(1) + self._port_to_br[port2] = br_src + self.add_linkpair( + Device(br_src, Device.BR_TYPE), + Device("OS Routing", Device.OS_ROUTE_TYPE), + Port(port), + Port("")) + elif re.search(r' \(internal\)', port): + port = re.sub(r' \(internal\)', '', port) + if port in self._iface_to_nss: + self.add_linkpair( + Device(br_src, Device.BR_TYPE), + Device(self._iface_to_nss[ + port], Device.NAMESPACE_TYPE), + Port(port), + Port(""), + self._formatter.NAMESPACE_FORMAT) + else: + self.add_linkpair( + Device(br_src, Device.BR_TYPE), + Device( + "INTERNAL", Device.OS_ROUTE_TYPE), + Port(port), + Port("")) + else: + ## Other OVSPort + self._port_to_br[port] = br_src + else: + continue + + def get_libvirt_conn(self): + if self._remote_desc is not None: + conn = libvirt.open("qemu+ssh://" + self._remote_desc + "/system") + else: + conn = libvirt.open("qemu:///system") + return conn + + def set_remote_sshkey(self, remote_sshkey): + self._runner.set_remote_sshkey(remote_sshkey) + + def set_remote_password(self, remote_password): + self._runner.set_remote_password(remote_password) + diff --git a/linkpair/collect/collector.pyc b/linkpair/collect/collector.pyc new file mode 100644 index 0000000..0bf67af Binary files /dev/null and b/linkpair/collect/collector.pyc differ diff --git a/linkpair/collector.py b/linkpair/collector.py index 6b8d59a..01e6229 100755 --- a/linkpair/collector.py +++ b/linkpair/collector.py @@ -18,7 +18,7 @@ from linkpair import LinkPair from formatter.grapheasy import GraphEasyFormatter from commonutils import CommonUtils from dbutils import DBUtils -from collectagent.commandrunner import CommandRunner +from collect.agent.commandrunner import CommandRunner class Collector(object): '''LinkpPair collector diff --git a/linkpair/collector.pyc b/linkpair/collector.pyc index c18b2a3..f913635 100644 Binary files a/linkpair/collector.pyc and b/linkpair/collector.pyc differ diff --git a/ti2.py b/ti2.py index 1a06a24..43b5e4e 100755 --- a/ti2.py +++ b/ti2.py @@ -12,7 +12,7 @@ import sys import os import re from optparse import OptionParser, OptionGroup -from linkpair.collector import Collector +from linkpair.collect.collector import Collector from linkpair.linkpair import LinkPair from linkpair.formatter.grapheasy import GraphEasyFormatter from linkpair.formatter.springy import SpringyFormatter