diff --git a/hyperglass/api/__init__.py b/hyperglass/api/__init__.py index 3b0e8b9..83acfb5 100644 --- a/hyperglass/api/__init__.py +++ b/hyperglass/api/__init__.py @@ -34,12 +34,11 @@ from hyperglass.api.models.response import ( ) WORKING_DIR = Path(__file__).parent -STATIC_DIR = WORKING_DIR.parent / "static" -IMAGES_DIR = STATIC_DIR / "images" EXAMPLES_DIR = WORKING_DIR / "examples" UI_DIR = STATIC_PATH / "ui" CUSTOM_DIR = STATIC_PATH / "custom" +IMAGES_DIR = STATIC_PATH / "images" EXAMPLE_DEVICES_PY = EXAMPLES_DIR / "devices.py" EXAMPLE_QUERIES_PY = EXAMPLES_DIR / "queries.py" diff --git a/hyperglass/util.py b/hyperglass/util.py index 2e0b7da..73d07f9 100644 --- a/hyperglass/util.py +++ b/hyperglass/util.py @@ -195,7 +195,69 @@ async def clear_redis_cache(db, config): return True -async def build_frontend(dev_mode, dev_url, prod_url, params, app_path, force=False): +async def move_files(src, dst, files): # noqa: C901 + """Move iterable of files from source to destination. + + Arguments: + src {Path} -- Current directory of files + dst {Path} -- Target destination directory + files {Iterable} -- Iterable of files + """ + import shutil + from pathlib import Path + from typing import Iterable + + def error(*args, **kwargs): + msg = ", ".join(args) + kwargs = {k: str(v) for k, v in kwargs.items()} + error_msg = msg.format(**kwargs) + log.error(error_msg) + return RuntimeError(error_msg) + + if not isinstance(src, Path): + try: + src = Path(src) + except TypeError: + raise error("{p} is not a valid path", p=src) + + if not isinstance(dst, Path): + try: + dst = Path(dst) + except TypeError: + raise error("{p} is not a valid path", p=dst) + + if not isinstance(files, Iterable): + raise error( + "{fa} must be an iterable (list, tuple, or generator). Received {f}", + fa="Files argument", + f=files, + ) + + for path in (src, dst): + if not path.exists(): + raise error("{p} does not exist", p=path) + + migrated = () + + for file in files: + dst_file = dst / file.name + + if not file.exists(): + raise error("{f} does not exist", f=file) + + try: + if not dst_file.exists(): + shutil.copyfile(file, dst_file) + migrated += (str(dst_file),) + except Exception as e: + raise error("Failed to migrate {f}: {e}", f=dst_file, e=e) + + return migrated + + +async def build_frontend( # noqa: C901 + dev_mode, dev_url, prod_url, params, app_path, force=False +): """Perform full frontend UI build process. Securely creates temporary file, writes frontend configuration @@ -223,6 +285,8 @@ async def build_frontend(dev_mode, dev_url, prod_url, params, app_path, force=Fa """ import hashlib import tempfile + import shutil + from filecmp import dircmp from pathlib import Path from aiofile import AIOFile import ujson as json @@ -297,6 +361,22 @@ async def build_frontend(dev_mode, dev_url, prod_url, params, app_path, force=Fa elif dev_mode and not force: log.debug("Running in developer mode, did not build new UI files") + """ + Compare repository's static assets with build directory's + assets. If the contents don't match, re-copy the files. + """ + asset_dir = Path(__file__).parent.parent / "assets" + target_dir = app_path / "static" / "images" + comparison = dircmp(asset_dir, target_dir, ignore=[".DS_Store"]) + + if not comparison.left_list == comparison.right_list: + shutil.copytree(asset_dir, target_dir) + if not comparison.left_list == comparison.right_list: + raise Exception( + "Files in '{a}' do not match files in '{b}'".format( + a=str(asset_dir), b=str(target_dir) + ) + ) except Exception as e: raise RuntimeError(str(e)) diff --git a/tests/ci_dev_server.py b/tests/ci_dev_server.py deleted file mode 100755 index 3f14c1b..0000000 --- a/tests/ci_dev_server.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python3 -""" -Starts hyperglass with the Sanic web server -""" -import os -import sys -import json -from logzero import logger - -working_directory = os.path.dirname(os.path.abspath(__file__)) -parent_directory = os.path.dirname(working_directory) - - -def construct_test(test_query, location, test_target): - """Constructs JSON POST data for test_hyperglass function.""" - constructed_query = json.dumps( - {"type": test_query, "location": location, "target": test_target} - ) - return constructed_query - - -def test_server(host, port): - """Starts Sanic web server for testing.""" - try: - sys.path.insert(0, parent_directory) - - from hyperglass import render - from hyperglass import hyperglass - - render.css() - logger.info("Starting Sanic web server...") - hyperglass.app.run(host=host, debug=True, port=port) - except: - logger.error("Exception occurred while trying to start test server...") - raise - - -if __name__ == "__main__": - test_server("localhost", 5000) diff --git a/tests/ci_prepare.py b/tests/ci_prepare.py deleted file mode 100755 index 1d9599c..0000000 --- a/tests/ci_prepare.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python3 -""" -Prepares the test environment prior to starting hyperglass. -""" -import os -import glob -import shutil -from logzero import logger - -working_directory = os.path.dirname(os.path.abspath(__file__)) -parent_directory = os.path.dirname(working_directory) - - -def ci_copy_config(): - """Copies test configuration files to usable config files""" - logger.info("Migrating test config files...") - config_dir = os.path.join(parent_directory, "hyperglass/configuration/") - test_files = glob.iglob(os.path.join(working_directory, "*.yaml")) - config_files = glob.iglob(os.path.join(config_dir, "*.yaml")) - logger.debug(config_dir) - logger.debug(working_directory) - logger.debug(parent_directory) - status = False - for file in config_files: - if os.path.exists(file): - logger.debug(f"{file} already exists") - os.remove(file) - logger.info(f"Deleted {file}") - for file in test_files: - try: - shutil.copy(file, config_dir) - logger.debug(f"Copied {file}") - logger.debug(os.listdir(config_dir)) - logger.info("Successfully migrated test config files") - status = True - except: - logger.error(f"Failed to migrate {file}") - raise - return status - - -if __name__ == "__main__": - ci_copy_config() diff --git a/tests/ci_test.py b/tests/ci_test.py deleted file mode 100755 index 0088863..0000000 --- a/tests/ci_test.py +++ /dev/null @@ -1,199 +0,0 @@ -#!/usr/bin/env python3 -""" -Runs tests against test hyperglass instance -""" -import asyncio -import os -import json -import http3 -import logzero - -working_directory = os.path.dirname(os.path.abspath(__file__)) -parent_directory = os.path.dirname(working_directory) - -# Async loop -loop = asyncio.get_event_loop() - -# Logzero Configuration -logger = logzero.logger -log_level = 10 -log_format = ( - "%(color)s[%(module)s:%(funcName)s:%(lineno)d " - "%(levelname)s]%(end_color)s %(message)s" -) -date_format = "%Y-%m-%d %H:%M:%S" -logzero_formatter = logzero.LogFormatter(fmt=log_format, datefmt=date_format) -logzero_config = logzero.setup_default_logger( - formatter=logzero_formatter, level=log_level -) - - -async def ci_hyperglass_test( - location, target_ipv4, target_ipv6, requires_ipv6_cidr, test_blacklist -): - """ - Tests hyperglass backend by making use of HTTP3 library to mimic - the JS Ajax POST performed by the front end. - """ - invalid_ip = "this_ain't_an_ip!" - invalid_aspath = ".*" - ipv4_cidr = "1.1.1.0/24" - ipv6_host = "2606:4700:4700::1111" - ipv6_cidr = "2606:4700:4700::/48" - test_headers = {"Content-Type": "application/json"} - test_endpoint = "http://localhost:5000/lg" - http_client = http3.AsyncClient() - # No Query Type Test - try: - logger.info("Starting No Query Type test...") - test_query = {"type": "", "location": location, "target": target_ipv4} - hg_response = await http_client.post( - test_endpoint, headers=test_headers, json=test_query - ) - if hg_response.status_code not in range(400, 500): - logger.error(hg_response.text) - raise RuntimeError("No Query Type test failed") - except: - logger.error("Exception occurred while running No Query Type test...") - raise - # No Location Test - try: - logger.info("Starting No Location test...") - test_query = {"type": "bgp_route", "location": "", "target": target_ipv6} - hg_response = await http_client.post( - test_endpoint, headers=test_headers, json=test_query - ) - if hg_response.status_code not in range(400, 500): - logger.error(hg_response.text) - raise RuntimeError("No Location test failed") - except: - logger.error("Exception occurred while running No Location test...") - raise - # No Target Test - try: - logger.info("Starting No Target test...") - test_query = {"type": "bgp_route", "location": location, "target": ""} - hg_response = await http_client.post( - test_endpoint, headers=test_headers, json=test_query - ) - if hg_response.status_code not in range(400, 500): - logger.error(hg_response.text) - raise RuntimeError("No Target test failed") - except: - logger.error("Exception occurred while running No Target test...") - raise - # Invalid BGP Route Test - try: - logger.info("Starting Invalid BGP IPv4 Route test...") - test_query = {"type": "bgp_route", "location": location, "target": invalid_ip} - hg_response = await http_client.post( - test_endpoint, headers=test_headers, json=test_query - ) - if hg_response.status_code not in range(400, 500): - logger.error(hg_response.text) - raise RuntimeError("Invalid BGP IPv4 Route test failed") - except: - logger.error("Exception occurred while running Invalid BGP IPv4 Route test...") - # Requires IPv6 CIDR Test - if requires_ipv6_cidr: - try: - logger.info("Starting Requires IPv6 CIDR test...") - test_query = { - "type": "bgp_route", - "location": requires_ipv6_cidr, - "target": ipv6_host, - } - hg_response = await http_client.post( - test_endpoint, headers=test_headers, json=test_query - ) - if hg_response.status_code not in range(400, 500): - logger.error(hg_response.text) - raise RuntimeError("Requires IPv6 CIDR test failed") - except: - logger.error("Exception occurred while running Requires IPv6 CIDR test...") - raise - # Invalid BGP Community Test - try: - logger.info("Starting Invalid BGP Community test...") - test_query = { - "type": "bgp_community", - "location": location, - "target": target_ipv4, - } - hg_response = await http_client.post( - test_endpoint, headers=test_headers, json=test_query - ) - if hg_response.status_code not in range(400, 500): - logger.error(hg_response.text) - raise RuntimeError("Invalid BGP Community test failed") - except: - logger.error("Exception occurred while running Invalid BGP Community test...") - raise - # Invalid BGP AS_PATH Test - try: - logger.info("Starting invalid BGP AS_PATH test...") - test_query = { - "type": "bgp_aspath", - "location": location, - "target": invalid_aspath, - } - hg_response = await http_client.post( - test_endpoint, headers=test_headers, json=test_query - ) - if hg_response.status_code not in range(400, 500): - logger.error(hg_response.text) - raise RuntimeError("Invalid BGP AS_PATH test failed") - except: - logger.error("Exception occurred while running Invalid BGP AS_PATH test...") - raise - # Invalid IPv4 Ping Test - try: - logger.info("Starting Invalid IPv4 Ping test...") - test_query = {"target": "ping", "location": location, "target": ipv4_cidr} - hg_response = await http_client.post( - test_endpoint, headers=test_headers, json=test_query - ) - if hg_response.status_code not in range(400, 500): - logger.error(hg_response.text) - raise RuntimeError("Invalid IPv4 Ping test failed") - except: - logger.error("Exception occurred while running Invalid IPv4 Ping test...") - raise - # Invalid IPv6 Ping Test - try: - logger.info("Starting Invalid IPv6 Ping test...") - test_query = {"type": "ping", "location": location, "target": ipv6_cidr} - hg_response = await http_client.post( - test_endpoint, headers=test_headers, json=test_query - ) - if hg_response.status_code not in range(400, 500): - logger.error(hg_response.text) - raise RuntimeError("Invalid IPv6 Ping test failed") - except: - logger.error("Exception occurred while running Invalid IPv6 Ping test...") - raise - # Blacklist Test - try: - logger.info("Starting Blacklist test...") - test_query = { - "type": "bgp_route", - "location": location, - "target": test_blacklist, - } - hg_response = await http_client.post( - test_endpoint, headers=test_headers, json=test_query - ) - if hg_response.status_code not in range(400, 500): - logger.error(hg_response.text) - raise RuntimeError("Blacklist test failed") - except: - logger.error("Exception occurred while running Blacklist test...") - raise - - -if __name__ == "__main__": - loop.run_until_complete( - ci_hyperglass_test( - "pop2", "1.1.1.0/24", "2606:4700:4700::/48", "pop1", "100.64.0.1" - ) - ) diff --git a/tests/devices.yaml b/tests/devices.yaml deleted file mode 100644 index 9cf649c..0000000 --- a/tests/devices.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# Routers -router: - test-r1: - address: 127.0.1.1 - asn: 65001 - src_addr_ipv4: 127.0.0.1 - src_addr_ipv6: ::1 - credential: default - location: pop1 - display_name: Test Router 1 - port: 22 - nos: cisco_ios - proxy: proxy1 - - test-r2: - address: 127.0.1.2 - asn: 65001 - src_addr_ipv4: 127.0.0.2 - src_addr_ipv6: ::1 - credential: default - location: pop2 - display_name: Test Router 2 - port: 22 - nos: cisco_xr - proxy: null -# Router Credentials -credential: - default: - username: username - password: password - - other: - username: otheradmin - password: otherpass -# SSH Proxy Servers -proxy: - proxy1: - address: 10.0.1.1 - username: username - password: password - nos: linux_ssh - ssh_command: ssh -l {username} {host} diff --git a/tests/hyperglass.yaml b/tests/hyperglass.yaml deleted file mode 100644 index f512c88..0000000 --- a/tests/hyperglass.yaml +++ /dev/null @@ -1,13 +0,0 @@ -general: - debug: true - primary_asn: 65001 - org_name: Travis CI Test -features: - rate_limit: - query: - rate: 1000 - site: - rate: 1000 - cache: - timeout: 1 - diff --git a/tests/requirements_dev.txt b/tests/requirements_dev.txt deleted file mode 100644 index e44aa8e..0000000 --- a/tests/requirements_dev.txt +++ /dev/null @@ -1,21 +0,0 @@ -anybadge==1.4.0 -black==19.3b0 -isort==4.3.21 -bandit==1.6.2 -flake8==3.7.8 -flake8-bandit==2.1.1 -flake8-black==0.1.0 -flake8-breakpoint==1.1.0 -flake8-bugbear==19.3.0 -flake8-builtins==1.4.1 -flake8-comprehensions==2.1.0 -flake8-deprecated==1.3 -flake8-eradicate==0.2.0 -flake8-if-expr==1.0.0 -flake8-isort==2.7.0 -flake8-pie==0.4.2 -flake8-plugin-utils==1.0.0 -flake8-polyfill==1.0.2 -flake8-print==3.1.0 -flake8-return==1.1.0 -pep8-naming==0.8.2