diff --git a/docs/configuration/index.md b/docs/configuration/configuration.md similarity index 100% rename from docs/configuration/index.md rename to docs/configuration/configuration.md diff --git a/docs/installation/reverseproxy.md b/docs/installation/reverseproxy.md index 7d8dbf4..c778c37 100644 --- a/docs/installation/reverseproxy.md +++ b/docs/installation/reverseproxy.md @@ -5,14 +5,25 @@ More than likely, you'll be exposing Hyperglass to the internet. It is recommend The below Nginx example assumes the default [Gunicorn](installation/wsgi) settings are used. ```nginx +geo $not_prometheus_hosts { + default 1; + 192.0.2.1/32 0; +} server { listen 80; - listen [::]:80ipv6only=on; + listen [::]:80 ipv6only=on; client_max_body_size 1024; server_name lg.domain.tld; + location /metrics { + if ($not_prometheus_hosts) { + rewrite /metrics /getyourownmetrics; + } + try_files $uri @proxy_to_app; + } + location /static/ { alias /opt/hyperglass/hyperglass/static/; } @@ -36,6 +47,10 @@ server { This configuration, in combination with the default Gunicorn configuration, makes the hyperglass front-end dual stack IPv4/IPv6 capable. To add SSL support, Nginx can be easily adjusted to terminate front-end SSL connections: ```nginx +geo $not_prometheus_hosts { + default 1; + 192.0.2.1/32 0; +} server { listen 80; listen [::]:80; @@ -53,6 +68,13 @@ server { server_name lg.domain.tld; + location /metrics { + if ($not_prometheus_hosts) { + rewrite /metrics /getyourownmetrics; + } + try_files $uri @proxy_to_app; + } + location /static/ { alias /opt/hyperglass/hyperglass/static/; } @@ -77,3 +99,6 @@ server { - Digital Ocean: [How To Secure Nginx with Let's Encrypt on Ubuntu 18.04](https://www.digitalocean.com/community/tutorials/how-to-secure-nginx-with-let-s-encrypt-on-ubuntu-18-04) - NGINX: [Using Free Let’s Encrypt SSL/TLS Certificates with NGINX](https://www.nginx.com/blog/using-free-ssltls-certificates-from-lets-encrypt-with-nginx/) + + +The `/metrics` block will ensure that hosts defined in the `geo $not_prometheus_hosts` directive are allowed to reach the `/metrics` URI, but that any other hosts will have the a request for `/metrics` rewritten to `/getyourownmetrics`, which will render the 404 error page. diff --git a/docs/installation/wsgi.md b/docs/installation/wsgi.md index ad21cca..d421228 100644 --- a/docs/installation/wsgi.md +++ b/docs/installation/wsgi.md @@ -14,21 +14,10 @@ $ pip3 install gunicorn Migrate the example Gunicorn configuration file: ```console $ cd /opt/hyperglass/ -$ python3 manage.py migrategunicorn +$ python3 manage.py migrate-gunicorn ``` -Open `hyperglass/gunicorn_config.py`, and adjust the parameters to match your local system. For example, make sure the `command` parameter matches the location of your `gunicorn` executable (`which gunicorn`), the `pythonpath` parameter matches the location where hyperglass is installed, and that the `user` parameter matches the user you're running hyperglass as: - -```python -import multiprocessing - -command = "/usr/local/bin/gunicorn" -pythonpath = "/opt/hyperglass/hyperglass" -bind = "[::1]:8001" -workers = multiprocessing.cpu_count() * 2 -user = "www-data" -timeout = 60 -``` +Open `hyperglass/hyperglass/gunicorn_config.py`, and adjust the parameters to match your local system. For example, make sure the `command` parameter matches the location of your `gunicorn` executable (`which gunicorn`), the `pythonpath` parameter matches the location where hyperglass is installed, and that the `user` parameter matches the user you're running hyperglass as: ### Permissions @@ -36,7 +25,7 @@ Gunicorn requires read/write/executable access to the entire `hyperglass/hypergl ```console # cd /opt/hyperglass/ -# python3 manage.py fixpermissions --user --group +# python3 manage.py update-permissions --user --group ``` !!! note "File Ownership" diff --git a/docs/monitoring.md b/docs/monitoring.md index e69de29..68f02c9 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -0,0 +1 @@ +Hyperglass has built in support for [Prometheus](https://prometheus.io/) metrics. diff --git a/file.test b/file.test deleted file mode 100644 index e69de29..0000000 diff --git a/hyperglass/.gitignore b/hyperglass/.gitignore index 644f839..27daead 100644 --- a/hyperglass/.gitignore +++ b/hyperglass/.gitignore @@ -3,6 +3,7 @@ .flask_cache .flask_cache/* gunicorn_config.py +gunicorn_dev_config.py test.py __pycache__/ parsing/ diff --git a/hyperglass/configuration/__init__.py b/hyperglass/configuration/__init__.py index 99634b5..ef3bedf 100644 --- a/hyperglass/configuration/__init__.py +++ b/hyperglass/configuration/__init__.py @@ -40,7 +40,7 @@ else: def blacklist(): """Returns list of subnets/IPs defined in blacklist.toml""" blacklist_config = config["blacklist"] - return blacklist_config["blacklist"] + return blacklist_config def requires_ipv6_cidr(nos): diff --git a/hyperglass/gunicorn_config.py.example b/hyperglass/gunicorn_config.py.example index 078b946..2cedbfe 100644 --- a/hyperglass/gunicorn_config.py.example +++ b/hyperglass/gunicorn_config.py.example @@ -2,8 +2,9 @@ https://github.com/checktheroads/hyperglass Guncorn configuration """ - +import os import multiprocessing +from logzero import logger command = "/usr/local/bin/gunicorn" pythonpath = "/opt/hyperglass" @@ -14,10 +15,36 @@ user = "www-data" timeout = 60 keepalive = 10 +# Prometheus Multiprocessing directory, set as environment variable +prometheus_multiproc_dir = ".prometheus_multiproc_dir" -def on_starting(server): - """Renders CSS templates at initial code execution with single worker""" - import hyperglass - hyperglass.render.css() - print(1) +def on_starting(server): # pylint: disable=unused-argument + """Pre-startup Gunicorn Tasks""" + try: + # Renders Jinja2 -> Sass, compiles Sass -> CSS prior to worker load + import hyperglass.render + + hyperglass.render.css() + print(1) + except ImportError as error_exception: + logger.error(f"Exception occurred:\n{error_exception}") + # + os.mkdir(prometheus_multiproc_dir) + os.environ["prometheus_multiproc_dir"] = prometheus_multiproc_dir + + +def worker_exit(server, worker): # pylint: disable=unused-argument + """Prometheus multiprocessing WSGI support""" + from prometheus_client import multiprocess + + multiprocess.mark_process_dead(worker.pid) + + +def on_exit(server): + try: + import shutil + except ImportError as error_exception: + logger.error(f"Exception occurred:\n{error_exception}") + + shutil.rmtree(prometheus_multiproc_dir) diff --git a/hyperglass/hyperglass.py b/hyperglass/hyperglass.py index d181465..aba95e9 100644 --- a/hyperglass/hyperglass.py +++ b/hyperglass/hyperglass.py @@ -14,7 +14,7 @@ from flask import Flask, request, Response from flask_caching import Cache from flask_limiter import Limiter from flask_limiter.util import get_ipaddr -from prometheus_client import generate_latest, Counter +from prometheus_client import generate_latest, Counter, CollectorRegistry, multiprocess # Project Imports from hyperglass.command import execute @@ -81,7 +81,9 @@ count_ratelimit = Counter( def metrics(): """Prometheus metrics""" content_type_latest = str("text/plain; version=0.0.4; charset=utf-8") - return Response(generate_latest(), mimetype=content_type_latest) + registry = CollectorRegistry() + multiprocess.MultiProcessCollector(registry) + return Response(generate_latest(registry), mimetype=content_type_latest) @app.errorhandler(404) diff --git a/hyperglass/hyperglass.service.example b/hyperglass/hyperglass.service.example index 222ac26..ab14a0b 100644 --- a/hyperglass/hyperglass.service.example +++ b/hyperglass/hyperglass.service.example @@ -1,5 +1,5 @@ [Unit] -Description=Hyperglass +Description=hyperglass After=network.target [Service] diff --git a/hyperglass/render/templates/404.html b/hyperglass/render/templates/404.html index bcaec09..ea5ae6d 100644 --- a/hyperglass/render/templates/404.html +++ b/hyperglass/render/templates/404.html @@ -10,10 +10,8 @@


diff --git a/hyperglass/render/templates/credit.html b/hyperglass/render/templates/credit.html index 9fb6da1..01ffc9c 100644 --- a/hyperglass/render/templates/credit.html +++ b/hyperglass/render/templates/credit.html @@ -1,3 +1,3 @@ -
+

Powered by hyperglass. Source code licensed BSD 3-Clause Clear.

diff --git a/hyperglass/render/templates/index.html b/hyperglass/render/templates/index.html index 3cc89b0..647a5cc 100644 --- a/hyperglass/render/templates/index.html +++ b/hyperglass/render/templates/index.html @@ -230,7 +230,7 @@
- + diff --git a/hyperglass/static/js/hyperglass.js b/hyperglass/static/js/hyperglass.js index cfcc4d2..20a759d 100644 --- a/hyperglass/static/js/hyperglass.js +++ b/hyperglass/static/js/hyperglass.js @@ -18,10 +18,10 @@ var btn_copy = document.getElementById('btn-copy'); var clipboard = new ClipboardJS(btn_copy); clipboard.on('success', function(e) { console.log(e); - $('#btn-copy').addClass('is-success'); + $('#btn-copy').addClass('is-success').addClass('is-outlined'); $('#copy-icon').removeClass('icofont-ui-copy').addClass('icofont-check'); setTimeout(function(){ - $('#btn-copy').removeClass('is-success'); + $('#btn-copy').removeClass('is-success').removeClass('is-outlined'); $('#copy-icon').removeClass('icofont-check').addClass('icofont-ui-copy'); }, 1000) }); diff --git a/manage.py b/manage.py index 17c83d4..6735e94 100755 --- a/manage.py +++ b/manage.py @@ -495,7 +495,7 @@ def compile_sass(): def migrateconfig(): """Copies example configuration files to usable config files""" try: - click.secho("Migrating example config files...", fg="cyan") + click.secho("Migrating example config files...", fg="black") hyperglass_root = os.path.dirname(hyperglass.__file__) config_dir = os.path.join(hyperglass_root, "configuration/") examples = glob.iglob(os.path.join(config_dir, "*.example")) @@ -523,7 +523,11 @@ def migrateconfig(): def migrategunicorn(): """Copies example Gunicorn config file to a usable config""" try: - click.secho("Migrating example Gunicorn configuration...", fg="cyan") + import hyperglass + except ImportError as error_exception: + click.secho(f"Error while importing hyperglass:\n{error_exception}", fg="red") + try: + click.secho("Migrating example Gunicorn configuration...", fg="black") hyperglass_root = os.path.dirname(hyperglass.__file__) ex_file = os.path.join(hyperglass_root, "gunicorn_config.py.example") basefile, extension = os.path.splitext(ex_file) @@ -533,15 +537,14 @@ def migrategunicorn(): else: try: cp(ex_file, newfile) - click.secho(f"✓ Migrated {newfile}", fg="green") + click.secho( + f"✓ Successfully migrated Gunicorn configuration to: {newfile}", + fg="green", + bold=True, + ) except: click.secho(f"✗ Failed to migrate {newfile}", fg="red") raise - click.secho( - "✓ Successfully migrated example Gunicorn configuration", - fg="green", - bold=True, - ) except: click.secho( "✗ Error migrating example Gunicorn configuration", fg="red", bold=True @@ -556,7 +559,7 @@ def migrategunicorn(): def migratesystemd(directory): """Copies example systemd service file to /etc/systemd/system/""" try: - click.secho("Migrating example systemd service...", fg="cyan") + click.secho("Migrating example systemd service...", fg="black") hyperglass_root = os.path.dirname(hyperglass.__file__) ex_file_base = "hyperglass.service.example" ex_file = os.path.join(hyperglass_root, ex_file_base) @@ -567,15 +570,14 @@ def migratesystemd(directory): else: try: cp(ex_file, newfile) - click.secho(f"✓ Migrated {newfile}", fg="green") + click.secho( + f"✓ Successfully migrated systemd service to: {newfile}", + fg="green", + bold=True, + ) except: click.secho(f"✗ Failed to migrate {newfile}", fg="red") raise - click.secho( - f"✓ Successfully migrated example systemd service to: {newfile}", - fg="green", - bold=True, - ) except: click.secho("✗ Error migrating example systemd service", fg="red", bold=True) raise diff --git a/mkdocs.yml b/mkdocs.yml index 8bd1b98..53776c9 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -10,7 +10,7 @@ nav: - 'Systemd': 'installation/systemd.md' - 'Reverse Proxy & SSL': 'installation/reverseproxy.md' - Configuration: - - 'Configuring Hyperglass': 'configuration/index.md' + - 'Configuring Hyperglass': 'configuration/configuration.md' - 'Devices': 'configuration/devices.md' - 'Branding': 'configuration/branding.md' - 'Features': 'configuration/features.md'