initial travis

.pylintrc

@@ -20,9 +20,6 @@
 # "duplicate-code" was disabled due to PyLint complaining about using the same logzero debug
 # configuration in two files. Apparently having a consistent logging configuration is "bad".
 
-[pre-commit-hook]
-limit=10.0
-
 [MASTER]
 
 # A comma-separated list of package or module names from where C extensions may

.travis.yml

@@ -5,10 +5,12 @@ before_install:
   - sudo apt-get update -q
   - sudo apt-get install -y redis
   - pip3 install black
+  - pip3 install pylint
 install:
   - pip3 install -r requirements.txt
-  - pip3 install gunicorn
-  - pip3 install pylint
 before_script:
-  - black hyperglass/
+  - bash ./ci/check_code.sh
-  - pylint hyperglass/
+script:
+  - python3 ./ci/ci_test.py
+after_script:
+  - bash ./ci/ci_commit.sh

README.md

@@ -7,8 +7,7 @@
 <br>
 
 ![GitHub issues](https://img.shields.io/github/issues/checktheroads/hyperglass.svg)
-![GitHub](https://img.shields.io/github/license/checktheroads/hyperglass.svg)
+![Pylint](https://github.com/checktheroads/hyperglass/blob/master/pylint.svg)
-![GitHub top language](https://img.shields.io/github/languages/top/checktheroads/hyperglass.svg)
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/ambv/black)
 
 ## Features

ci/check_code.sh

@@ -13,13 +13,13 @@ check_format() {
 }
 
 run_pylint() {
-  python3 manage.py pylint-badge --integer-only True
+  echo $(python3 manage.py pylint-badge --integer-only True)
 }
 
 check_pylint() {
   PYLINT_SCORE=$(run_pylint)
   echo "Pylint score: $PYLINT_SCORE"
-  if  [ "$PYLINT_SCORE" != "10.00" ]
+  if  [ "$PYLINT_SCORE" == "10.00" ]
   then
     git add pylint.svg
     git commit --message "Pylint Badge - travis #$TRAVIS_BUILD_NUMBER"
@@ -27,11 +27,5 @@ check_pylint() {
   fi
 }
 
-# upload_files() {
-#   git remote add origin-pages https://${GH_TOKEN}@github.com/MVSE-outreach/resources.git > /dev/null 2>&1
-#   git push --quiet --set-upstream origin-pages gh-pages
-# }
-
 check_format
 check_pylint
-# setup_git

ci/ci_commit.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+git_commit() {
+  git remote add origin https://${GH_TOKEN}@github.com/checktheroads/hyperglass.git > /dev/null 2>&1
+  git push --quiet --set-upstream origin master
+}
+
+git_commit()

ci/ci_test.py

@@ -0,0 +1,209 @@
+from logzero import logger
+
+working_directory = os.path.dirname(os.path.abspath(__file__))
+
+
+def construct_test(test_query, location, test_target):
+    """Constructs JSON POST data for test_hyperglass function"""
+    constructed_query = json.dumps(
+        {"type": test_query, "location": location, "target": test_target}
+    )
+    return constructed_query
+
+
+def ci_test(
+    location,
+    target_ipv4,
+    target_ipv6,
+    requires_ipv6_cidr,
+    test_blacklist,
+    test_community,
+    test_aspath,
+    test_host,
+    test_port,
+):
+    """Fully tests hyperglass backend by making use of requests library to mimic the JS Ajax POST \
+    performed by the front end."""
+    test_target = None
+    invalid_ip = "this_ain't_an_ip!"
+    invalid_community = "192.0.2.1"
+    invalid_aspath = ".*"
+    ipv4_host = "1.1.1.1"
+    ipv4_cidr = "1.1.1.0/24"
+    ipv6_host = "2606:4700:4700::1111"
+    ipv6_cidr = "2606:4700:4700::/48"
+    test_headers = {"Content-Type": "application/json"}
+    test_endpoint = f"http://{test_host}:{test_port}/lg"
+    # No Query Type Test
+    try:
+        logger.info("Starting No Query Type test...")
+        test_query = construct_test("", location, target_ipv4)
+        hg_response = requests.post(
+            test_endpoint, headers=test_headers, data=test_query
+        )
+        if not hg_response.status_code in range(400, 500):
+            raise RuntimeError("No Query Type test failed")
+    except:
+        logger.error("Exception occurred while running No Query Type test...")
+        raise
+    # No Location Test
+    try:
+        logger.info("Starting No Location test...")
+        test_query = construct_test("bgp_route", "", target_ipv6)
+        hg_response = requests.post(
+            test_endpoint, headers=test_headers, data=test_query
+        )
+        if not hg_response.status_code in range(400, 500):
+            raise RuntimeError("No Location test failed")
+    except:
+        logger.error("Exception occurred while running No Location test...")
+        raise
+    # No Target Test
+    try:
+        logger.info("Starting No Target test...")
+        test_query = construct_test("bgp_route", location, "")
+        hg_response = requests.post(
+            test_endpoint, headers=test_headers, data=test_query
+        )
+        if not hg_response.status_code in range(400, 500):
+            raise RuntimeError("No Target test failed")
+    except:
+        logger.error("Exception occurred while running No Target test...")
+        raise
+    # Invalid BGP Route Test
+    try:
+        logger.info("Starting Invalid BGP IPv4 Route test...")
+        test_query = construct_test("bgp_route", location, invalid_ip)
+        hg_response = requests.post(
+            test_endpoint, headers=test_headers, data=test_query
+        )
+        if not hg_response.status_code in range(400, 500):
+            raise RuntimeError("Invalid BGP IPv4 Route test failed")
+    except:
+        logger.error("Exception occurred while running Invalid BGP IPv4 Route test...")
+    # Requires IPv6 CIDR Test
+    if requires_ipv6_cidr:
+        try:
+            logger.info("Starting Requires IPv6 CIDR test...")
+            test_query = construct_test("bgp_route", requires_ipv6_cidr, ipv6_host)
+            hg_response = requests.post(
+                test_endpoint, headers=test_headers, data=test_query
+            )
+            if not hg_response.status_code in range(400, 500):
+                raise RuntimeError("Requires IPv6 CIDR test failed")
+        except:
+            logger.error("Exception occurred while running Requires IPv6 CIDR test...")
+            raise
+    # Invalid BGP Community Test
+    try:
+        logger.info("Starting Invalid BGP Community test...")
+        test_query = construct_test("bgp_community", location, target_ipv4)
+        hg_response = requests.post(
+            test_endpoint, headers=test_headers, data=test_query
+        )
+        if not hg_response.status_code in range(400, 500):
+            raise RuntimeError("Invalid BGP Community test failed")
+    except:
+        logger.error("Exception occurred while running Invalid BGP Community test...")
+        raise
+    # Invalid BGP AS_PATH Test
+    try:
+        logger.info("Starting invalid BGP AS_PATH test...")
+        test_query = construct_test("bgp_aspath", location, invalid_aspath)
+        hg_response = requests.post(
+            test_endpoint, headers=test_headers, data=test_query
+        )
+        if not hg_response.status_code in range(400, 500):
+            raise RuntimeError("Invalid BGP AS_PATH test failed")
+    except:
+        logger.error("Exception occurred while running Invalid BGP AS_PATH test...")
+        raise
+    # Invalid IPv4 Ping Test
+    try:
+        logger.info("Starting Invalid IPv4 Ping test...")
+        test_query = construct_test("ping", location, ipv4_cidr)
+        hg_response = requests.post(
+            test_endpoint, headers=test_headers, data=test_query
+        )
+        if not hg_response.status_code in range(400, 500):
+            raise RuntimeError("Invalid IPv4 Ping test failed")
+    except:
+        logger.error("Exception occurred while running Invalid IPv4 Ping test...")
+        raise
+    # Invalid IPv6 Ping Test
+    try:
+        logger.info("Starting Invalid IPv6 Ping test...")
+        test_query = construct_test("ping", location, ipv6_cidr)
+        hg_response = requests.post(
+            test_endpoint, headers=test_headers, data=test_query
+        )
+        if not hg_response.status_code in range(400, 500):
+            raise RuntimeError("Invalid IPv6 Ping test failed")
+    except:
+        logger.error("Exception occurred while running Invalid IPv6 Ping test...")
+        raise
+    # Blacklist Test
+    try:
+        logger.info("Starting Blacklist test...")
+        test_query = construct_test("bgp_route", location, test_blacklist)
+        hg_response = requests.post(
+            test_endpoint, headers=test_headers, data=test_query
+        )
+        if not hg_response.status_code in range(400, 500):
+            raise RuntimeError("Blacklist test failed")
+    except:
+        logger.error("Exception occurred while running Blacklist test...")
+        raise
+
+
+def ci_config():
+    """Copies test configuration files to usable config files"""
+    try:
+        logger.info("Migrating test config files...")
+        config_dir = os.path.join(working_directory, "hyperglass/configuration/")
+        ci_dir = os.path.join(working_directory, "ci/")
+        test_files = glob.iglob(os.path.join(ci_dir, "*.toml"))
+        for f in test_files:
+            if os.path.exists(f):
+                raise RuntimeError(f"{f} already exists")
+            else:
+                try:
+                    cp(f, config_dir)
+                    logger.info("Migrated test config files")
+                except:
+                    logger.error(f"Failed to migrate {f}")
+                    raise
+    except:
+        logger.error("Error migrating test config files")
+        raise
+
+
+def flask_dev_server(host, port):
+    """Starts Flask development server for testing without WSGI/Reverse Proxy"""
+    try:
+        from hyperglass import hyperglass
+        from hyperglass import configuration
+        from hyperglass import render
+
+        render.css()
+        logger.info("Starting Flask development server")
+        hyperglass.app.run(host=host, debug=True, port=port)
+    except:
+        logger.error("Exception occurred while trying to start test server...")
+        raise
+
+
+if __name__ == "__main__":
+    ci_config()
+    flask_dev_server("localhost", 5000)
+    ci_test(
+        "pop2",
+        "1.1.1.0/24",
+        "2606:4700:4700::/48",
+        "pop1",
+        "100.64.0.1",
+        "65001:1",
+        "_65001$",
+        "localhost",
+        5000,
+    )

ci/commands.toml

@@ -0,0 +1,38 @@
+[[cisco_ios]]
+[cisco_ios.dual]
+bgp_community = "show bgp all community {target}"
+bgp_aspath = 'show bgp all quote-regexp "{target}"'
+[cisco_ios.ipv4]
+bgp_route = "show bgp ipv4 unicast {target} | exclude pathid:|Epoch"
+ping = "ping {target} repeat 5 source {source}"
+traceroute = "traceroute {target} timeout 1 probe 2 source {source}"
+[cisco_ios.ipv6]
+bgp_route = "show bgp ipv6 unicast {target} | exclude pathid:|Epoch"
+ping = "ping ipv6 {target} repeat 5 source {source}"
+traceroute = "traceroute ipv6 {target} timeout 1 probe 2 source {source}"
+
+[[cisco_xr]]
+[cisco_xr.dual]
+bgp_community = 'show bgp all unicast community {target} | utility egrep -v "\(BGP |Table |Non-stop\)"'
+bgp_aspath = 'show bgp all unicast regexp {target} | utility egrep -v "\(BGP |Table |Non-stop\)"'
+[cisco_xr.ipv4]
+bgp_route = 'show bgp ipv4 unicast {target} | util egrep "\(BGP routing table entry|Path \#|aggregated by|Origin |Community:|validity| from \)"'
+ping = "ping ipv4 {target} count 5 source {src_addr_ipv4}"
+traceroute = "traceroute ipv4 {target} timeout 1 probe 2 source {source}"
+[cisco_xr.ipv6]
+bgp_route = 'show bgp ipv6 unicast {target} | util egrep "\(BGP routing table entry|Path \#|aggregated by|Origin |Community:|validity| from \)"'
+ping = "ping ipv6 {target} count 5 source {src_addr_ipv6}"
+traceroute = "traceroute ipv6 {target} timeout 1 probe 2 source {source}"
+
+[[juniper]]
+[juniper.dual]
+bgp_community = "show route protocol bgp community {target}"
+bgp_aspath = "show route protocol bgp aspath-regex {target}"
+[juniper.ipv4]
+bgp_route = "show route protocol bgp table inet.0 {target} detail"
+ping = "ping inet {target} count 5 source {src_addr_ipv4}"
+traceroute = "traceroute inet {target} wait 1 source {source}"
+[juniper.ipv6]
+bgp_route = "show route protocol bgp table inet6.0 {target} detail"
+ping = "ping inet6 {target} count 5 source {src_addr_ipv6}"
+traceroute = "traceroute inet6 {target} wait 1 source {source}"

ci/configuration.toml

@@ -0,0 +1,122 @@
+# Non-dictionary parameters
+debug = false
+requires_ipv6_cidr = [
+"cisco_ios",
+"cisco_nxos"
+]
+# IP Blacklist
+blacklist = [
+"198.18.0.0/15",
+"100.64.0.0/10",
+"2001:db8::/32",
+"10.0.0.0/8",
+"192.168.0.0/16",
+"172.16.0.0/12"
+]
+# General site-wide parameters
+[general]
+primary_asn = "65001"
+org_name = "Travis CI Test"
+# google_analytics = ""
+
+# Feature customization
+[features]
+[features.rate_limit.query]
+rate = 1000
+# title = ""
+# message = ""
+# button = ""
+[features.rate_limit.site]
+rate = 1000
+# title = ""
+# subtitle = ""
+[features.cache]
+timeout = 1
+# directory = ""
+# show_text = true
+# text = ""
+[features.bgp_route]
+# enable = true
+[features.bgp_community]
+# enable = true
+[features.bgp_community.regex]
+# decimal = ""
+# extended_as = ""
+# large = ""
+[features.bgp_aspath]
+# enable = true
+[features.bgp_aspath.regex]
+# mode = ""
+# asplain = ""
+# asdot = ""
+[features.ping]
+# enable = true
+[features.traceroute]
+# enable = true
+[features.max_prefix]
+# enable = false
+# ipv4 = 24
+# ipv6 = 64
+# message = ""
+
+# User messages
+[messages]
+# no_query_type = ""
+# no_location = ""
+# no_input = ""
+# not_allowed = ""
+# requires_ipv6_cidr = ""
+# invalid_ip = ""
+# invalid_dual = ""
+# general = ""
+# directed_cidr = ""
+
+# Branding/Visual Customization Parameters
+[branding]
+# site_name = ""
+[branding.footer]
+# enable = true
+[branding.credit]
+# enable = true
+[branding.peering_db]
+# enable = true
+[branding.text]
+# title_mode = "text_only"
+# title = ""
+# subtitle = ""
+# query_type = ""
+# results = ""
+# location = ""
+# query_placeholder = ""
+# bgp_route = ""
+# bgp_community = ""
+# bgp_aspath = ""
+# ping = ""
+# traceroute = ""
+[branding.text.404]
+# title = ""
+# subtitle = ""
+[branding.text.500]
+# title = ""
+# subtitle = ""
+# button = ""
+[branding.logo]
+# path = ""
+# width = ""
+# favicons = ""
+[branding.color]
+# background = ""
+# button_submit = ""
+# danger = ""
+# progress_bar = ""
+[branding.color.tag]
+# type = ""
+# location = ""
+# location_title = ""
+# type_title = ""
+[branding.font.primary]
+# name = ""
+# url = ""
+[branding.font.mono]
+# name = ""
+# url = ""

ci/devices.toml

@@ -0,0 +1,43 @@
+# Routers
+[router.'test-r1']
+address = "127.0.1.1"
+asn = "65001"
+src_addr_ipv4 = "127.0.0.1"
+src_addr_ipv6 = "::1"
+credential = "default"
+location = "pop1"
+name = "test-r1.pop1"
+display_name = "Test Router 1"
+port = "22"
+type = "cisco_ios"
+proxy = "proxy1"
+
+[router.'test-r2']
+address = "127.0.1.2"
+asn = "65001"
+src_addr_ipv4 = "127.0.0.2"
+src_addr_ipv6 = "::1"
+credential = "default"
+location = "pop2"
+name = "test-r2.pop2"
+display_name = "Test Router 2"
+port = "22"
+type = "cisco_xr"
+proxy = ""
+
+# Router Credentials
+[credential.'default']
+username = "username"
+password = "password"
+
+[credential.'other']
+username = "otheradmin"
+password = "otherpass"
+
+# SSH Proxy Servers
+[proxy.'proxy1']
+address = "10.0.1.1"
+username = "username"
+password = "password"
+type = "linux_ssh"
+ssh_command = "ssh -l {username} {host}"

hyperglass/configuration/devices.toml.example

@@ -8,7 +8,7 @@ credential = "default"
 location = "pop1"
 name = "router1.pop1"
 display_name = "Router"
-port = "222"
+port = "22"
 type = "cisco_ios"
 proxy = "proxy1"