diff --git a/.pylintrc b/.pylintrc index 9bb57ed..aa11923 100644 --- a/.pylintrc +++ b/.pylintrc @@ -20,9 +20,6 @@ # "duplicate-code" was disabled due to PyLint complaining about using the same logzero debug # configuration in two files. Apparently having a consistent logging configuration is "bad". -[pre-commit-hook] -limit=10.0 - [MASTER] # A comma-separated list of package or module names from where C extensions may diff --git a/.travis.yml b/.travis.yml index db59eab..ffc6f9e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,10 +5,12 @@ before_install: - sudo apt-get update -q - sudo apt-get install -y redis - pip3 install black + - pip3 install pylint install: - pip3 install -r requirements.txt - - pip3 install gunicorn - - pip3 install pylint before_script: - - black hyperglass/ - - pylint hyperglass/ + - bash ./ci/check_code.sh +script: + - python3 ./ci/ci_test.py +after_script: + - bash ./ci/ci_commit.sh diff --git a/README.md b/README.md index 3411fbb..7f522de 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,7 @@
![GitHub issues](https://img.shields.io/github/issues/checktheroads/hyperglass.svg) -![GitHub](https://img.shields.io/github/license/checktheroads/hyperglass.svg) -![GitHub top language](https://img.shields.io/github/languages/top/checktheroads/hyperglass.svg) +![Pylint](https://github.com/checktheroads/hyperglass/blob/master/pylint.svg) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/ambv/black) ## Features diff --git a/ci/push.sh b/ci/check_code.sh similarity index 66% rename from ci/push.sh rename to ci/check_code.sh index 30082db..5c13805 100755 --- a/ci/push.sh +++ b/ci/check_code.sh @@ -13,13 +13,13 @@ check_format() { } run_pylint() { - python3 manage.py pylint-badge --integer-only True + echo $(python3 manage.py pylint-badge --integer-only True) } check_pylint() { PYLINT_SCORE=$(run_pylint) echo "Pylint score: $PYLINT_SCORE" - if [ "$PYLINT_SCORE" != "10.00" ] + if [ "$PYLINT_SCORE" == "10.00" ] then git add pylint.svg git commit --message "Pylint Badge - travis #$TRAVIS_BUILD_NUMBER" @@ -27,11 +27,5 @@ check_pylint() { fi } -# upload_files() { -# git remote add origin-pages https://${GH_TOKEN}@github.com/MVSE-outreach/resources.git > /dev/null 2>&1 -# git push --quiet --set-upstream origin-pages gh-pages -# } - check_format check_pylint -# setup_git diff --git a/ci/ci_commit.sh b/ci/ci_commit.sh new file mode 100644 index 0000000..865a7cd --- /dev/null +++ b/ci/ci_commit.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +git_commit() { + git remote add origin https://${GH_TOKEN}@github.com/checktheroads/hyperglass.git > /dev/null 2>&1 + git push --quiet --set-upstream origin master +} + +git_commit() diff --git a/ci/ci_test.py b/ci/ci_test.py new file mode 100644 index 0000000..4752a1a --- /dev/null +++ b/ci/ci_test.py @@ -0,0 +1,209 @@ +from logzero import logger + +working_directory = os.path.dirname(os.path.abspath(__file__)) + + +def construct_test(test_query, location, test_target): + """Constructs JSON POST data for test_hyperglass function""" + constructed_query = json.dumps( + {"type": test_query, "location": location, "target": test_target} + ) + return constructed_query + + +def ci_test( + location, + target_ipv4, + target_ipv6, + requires_ipv6_cidr, + test_blacklist, + test_community, + test_aspath, + test_host, + test_port, +): + """Fully tests hyperglass backend by making use of requests library to mimic the JS Ajax POST \ + performed by the front end.""" + test_target = None + invalid_ip = "this_ain't_an_ip!" + invalid_community = "192.0.2.1" + invalid_aspath = ".*" + ipv4_host = "1.1.1.1" + ipv4_cidr = "1.1.1.0/24" + ipv6_host = "2606:4700:4700::1111" + ipv6_cidr = "2606:4700:4700::/48" + test_headers = {"Content-Type": "application/json"} + test_endpoint = f"http://{test_host}:{test_port}/lg" + # No Query Type Test + try: + logger.info("Starting No Query Type test...") + test_query = construct_test("", location, target_ipv4) + hg_response = requests.post( + test_endpoint, headers=test_headers, data=test_query + ) + if not hg_response.status_code in range(400, 500): + raise RuntimeError("No Query Type test failed") + except: + logger.error("Exception occurred while running No Query Type test...") + raise + # No Location Test + try: + logger.info("Starting No Location test...") + test_query = construct_test("bgp_route", "", target_ipv6) + hg_response = requests.post( + test_endpoint, headers=test_headers, data=test_query + ) + if not hg_response.status_code in range(400, 500): + raise RuntimeError("No Location test failed") + except: + logger.error("Exception occurred while running No Location test...") + raise + # No Target Test + try: + logger.info("Starting No Target test...") + test_query = construct_test("bgp_route", location, "") + hg_response = requests.post( + test_endpoint, headers=test_headers, data=test_query + ) + if not hg_response.status_code in range(400, 500): + raise RuntimeError("No Target test failed") + except: + logger.error("Exception occurred while running No Target test...") + raise + # Invalid BGP Route Test + try: + logger.info("Starting Invalid BGP IPv4 Route test...") + test_query = construct_test("bgp_route", location, invalid_ip) + hg_response = requests.post( + test_endpoint, headers=test_headers, data=test_query + ) + if not hg_response.status_code in range(400, 500): + raise RuntimeError("Invalid BGP IPv4 Route test failed") + except: + logger.error("Exception occurred while running Invalid BGP IPv4 Route test...") + # Requires IPv6 CIDR Test + if requires_ipv6_cidr: + try: + logger.info("Starting Requires IPv6 CIDR test...") + test_query = construct_test("bgp_route", requires_ipv6_cidr, ipv6_host) + hg_response = requests.post( + test_endpoint, headers=test_headers, data=test_query + ) + if not hg_response.status_code in range(400, 500): + raise RuntimeError("Requires IPv6 CIDR test failed") + except: + logger.error("Exception occurred while running Requires IPv6 CIDR test...") + raise + # Invalid BGP Community Test + try: + logger.info("Starting Invalid BGP Community test...") + test_query = construct_test("bgp_community", location, target_ipv4) + hg_response = requests.post( + test_endpoint, headers=test_headers, data=test_query + ) + if not hg_response.status_code in range(400, 500): + raise RuntimeError("Invalid BGP Community test failed") + except: + logger.error("Exception occurred while running Invalid BGP Community test...") + raise + # Invalid BGP AS_PATH Test + try: + logger.info("Starting invalid BGP AS_PATH test...") + test_query = construct_test("bgp_aspath", location, invalid_aspath) + hg_response = requests.post( + test_endpoint, headers=test_headers, data=test_query + ) + if not hg_response.status_code in range(400, 500): + raise RuntimeError("Invalid BGP AS_PATH test failed") + except: + logger.error("Exception occurred while running Invalid BGP AS_PATH test...") + raise + # Invalid IPv4 Ping Test + try: + logger.info("Starting Invalid IPv4 Ping test...") + test_query = construct_test("ping", location, ipv4_cidr) + hg_response = requests.post( + test_endpoint, headers=test_headers, data=test_query + ) + if not hg_response.status_code in range(400, 500): + raise RuntimeError("Invalid IPv4 Ping test failed") + except: + logger.error("Exception occurred while running Invalid IPv4 Ping test...") + raise + # Invalid IPv6 Ping Test + try: + logger.info("Starting Invalid IPv6 Ping test...") + test_query = construct_test("ping", location, ipv6_cidr) + hg_response = requests.post( + test_endpoint, headers=test_headers, data=test_query + ) + if not hg_response.status_code in range(400, 500): + raise RuntimeError("Invalid IPv6 Ping test failed") + except: + logger.error("Exception occurred while running Invalid IPv6 Ping test...") + raise + # Blacklist Test + try: + logger.info("Starting Blacklist test...") + test_query = construct_test("bgp_route", location, test_blacklist) + hg_response = requests.post( + test_endpoint, headers=test_headers, data=test_query + ) + if not hg_response.status_code in range(400, 500): + raise RuntimeError("Blacklist test failed") + except: + logger.error("Exception occurred while running Blacklist test...") + raise + + +def ci_config(): + """Copies test configuration files to usable config files""" + try: + logger.info("Migrating test config files...") + config_dir = os.path.join(working_directory, "hyperglass/configuration/") + ci_dir = os.path.join(working_directory, "ci/") + test_files = glob.iglob(os.path.join(ci_dir, "*.toml")) + for f in test_files: + if os.path.exists(f): + raise RuntimeError(f"{f} already exists") + else: + try: + cp(f, config_dir) + logger.info("Migrated test config files") + except: + logger.error(f"Failed to migrate {f}") + raise + except: + logger.error("Error migrating test config files") + raise + + +def flask_dev_server(host, port): + """Starts Flask development server for testing without WSGI/Reverse Proxy""" + try: + from hyperglass import hyperglass + from hyperglass import configuration + from hyperglass import render + + render.css() + logger.info("Starting Flask development server") + hyperglass.app.run(host=host, debug=True, port=port) + except: + logger.error("Exception occurred while trying to start test server...") + raise + + +if __name__ == "__main__": + ci_config() + flask_dev_server("localhost", 5000) + ci_test( + "pop2", + "1.1.1.0/24", + "2606:4700:4700::/48", + "pop1", + "100.64.0.1", + "65001:1", + "_65001$", + "localhost", + 5000, + ) diff --git a/ci/commands.toml b/ci/commands.toml new file mode 100644 index 0000000..78e9145 --- /dev/null +++ b/ci/commands.toml @@ -0,0 +1,38 @@ +[[cisco_ios]] +[cisco_ios.dual] +bgp_community = "show bgp all community {target}" +bgp_aspath = 'show bgp all quote-regexp "{target}"' +[cisco_ios.ipv4] +bgp_route = "show bgp ipv4 unicast {target} | exclude pathid:|Epoch" +ping = "ping {target} repeat 5 source {source}" +traceroute = "traceroute {target} timeout 1 probe 2 source {source}" +[cisco_ios.ipv6] +bgp_route = "show bgp ipv6 unicast {target} | exclude pathid:|Epoch" +ping = "ping ipv6 {target} repeat 5 source {source}" +traceroute = "traceroute ipv6 {target} timeout 1 probe 2 source {source}" + +[[cisco_xr]] +[cisco_xr.dual] +bgp_community = 'show bgp all unicast community {target} | utility egrep -v "\(BGP |Table |Non-stop\)"' +bgp_aspath = 'show bgp all unicast regexp {target} | utility egrep -v "\(BGP |Table |Non-stop\)"' +[cisco_xr.ipv4] +bgp_route = 'show bgp ipv4 unicast {target} | util egrep "\(BGP routing table entry|Path \#|aggregated by|Origin |Community:|validity| from \)"' +ping = "ping ipv4 {target} count 5 source {src_addr_ipv4}" +traceroute = "traceroute ipv4 {target} timeout 1 probe 2 source {source}" +[cisco_xr.ipv6] +bgp_route = 'show bgp ipv6 unicast {target} | util egrep "\(BGP routing table entry|Path \#|aggregated by|Origin |Community:|validity| from \)"' +ping = "ping ipv6 {target} count 5 source {src_addr_ipv6}" +traceroute = "traceroute ipv6 {target} timeout 1 probe 2 source {source}" + +[[juniper]] +[juniper.dual] +bgp_community = "show route protocol bgp community {target}" +bgp_aspath = "show route protocol bgp aspath-regex {target}" +[juniper.ipv4] +bgp_route = "show route protocol bgp table inet.0 {target} detail" +ping = "ping inet {target} count 5 source {src_addr_ipv4}" +traceroute = "traceroute inet {target} wait 1 source {source}" +[juniper.ipv6] +bgp_route = "show route protocol bgp table inet6.0 {target} detail" +ping = "ping inet6 {target} count 5 source {src_addr_ipv6}" +traceroute = "traceroute inet6 {target} wait 1 source {source}" diff --git a/ci/configuration.toml b/ci/configuration.toml new file mode 100644 index 0000000..e00684c --- /dev/null +++ b/ci/configuration.toml @@ -0,0 +1,122 @@ +# Non-dictionary parameters +debug = false +requires_ipv6_cidr = [ +"cisco_ios", +"cisco_nxos" +] +# IP Blacklist +blacklist = [ +"198.18.0.0/15", +"100.64.0.0/10", +"2001:db8::/32", +"10.0.0.0/8", +"192.168.0.0/16", +"172.16.0.0/12" +] +# General site-wide parameters +[general] +primary_asn = "65001" +org_name = "Travis CI Test" +# google_analytics = "" + +# Feature customization +[features] +[features.rate_limit.query] +rate = 1000 +# title = "" +# message = "" +# button = "" +[features.rate_limit.site] +rate = 1000 +# title = "" +# subtitle = "" +[features.cache] +timeout = 1 +# directory = "" +# show_text = true +# text = "" +[features.bgp_route] +# enable = true +[features.bgp_community] +# enable = true +[features.bgp_community.regex] +# decimal = "" +# extended_as = "" +# large = "" +[features.bgp_aspath] +# enable = true +[features.bgp_aspath.regex] +# mode = "" +# asplain = "" +# asdot = "" +[features.ping] +# enable = true +[features.traceroute] +# enable = true +[features.max_prefix] +# enable = false +# ipv4 = 24 +# ipv6 = 64 +# message = "" + +# User messages +[messages] +# no_query_type = "" +# no_location = "" +# no_input = "" +# not_allowed = "" +# requires_ipv6_cidr = "" +# invalid_ip = "" +# invalid_dual = "" +# general = "" +# directed_cidr = "" + +# Branding/Visual Customization Parameters +[branding] +# site_name = "" +[branding.footer] +# enable = true +[branding.credit] +# enable = true +[branding.peering_db] +# enable = true +[branding.text] +# title_mode = "text_only" +# title = "" +# subtitle = "" +# query_type = "" +# results = "" +# location = "" +# query_placeholder = "" +# bgp_route = "" +# bgp_community = "" +# bgp_aspath = "" +# ping = "" +# traceroute = "" +[branding.text.404] +# title = "" +# subtitle = "" +[branding.text.500] +# title = "" +# subtitle = "" +# button = "" +[branding.logo] +# path = "" +# width = "" +# favicons = "" +[branding.color] +# background = "" +# button_submit = "" +# danger = "" +# progress_bar = "" +[branding.color.tag] +# type = "" +# location = "" +# location_title = "" +# type_title = "" +[branding.font.primary] +# name = "" +# url = "" +[branding.font.mono] +# name = "" +# url = "" diff --git a/ci/devices.toml b/ci/devices.toml new file mode 100644 index 0000000..5b47780 --- /dev/null +++ b/ci/devices.toml @@ -0,0 +1,43 @@ +# Routers +[router.'test-r1'] +address = "127.0.1.1" +asn = "65001" +src_addr_ipv4 = "127.0.0.1" +src_addr_ipv6 = "::1" +credential = "default" +location = "pop1" +name = "test-r1.pop1" +display_name = "Test Router 1" +port = "22" +type = "cisco_ios" +proxy = "proxy1" + +[router.'test-r2'] +address = "127.0.1.2" +asn = "65001" +src_addr_ipv4 = "127.0.0.2" +src_addr_ipv6 = "::1" +credential = "default" +location = "pop2" +name = "test-r2.pop2" +display_name = "Test Router 2" +port = "22" +type = "cisco_xr" +proxy = "" + +# Router Credentials +[credential.'default'] +username = "username" +password = "password" + +[credential.'other'] +username = "otheradmin" +password = "otherpass" + +# SSH Proxy Servers +[proxy.'proxy1'] +address = "10.0.1.1" +username = "username" +password = "password" +type = "linux_ssh" +ssh_command = "ssh -l {username} {host}" diff --git a/hyperglass/configuration/devices.toml.example b/hyperglass/configuration/devices.toml.example index 5be3373..a0259eb 100644 --- a/hyperglass/configuration/devices.toml.example +++ b/hyperglass/configuration/devices.toml.example @@ -8,7 +8,7 @@ credential = "default" location = "pop1" name = "router1.pop1" display_name = "Router" -port = "222" +port = "22" type = "cisco_ios" proxy = "proxy1"