Commit e6de58db authored by Lukas Burgey's avatar Lukas Burgey

Merge branch 'dev'

parents fbee6a81 b343b80b
.secret.key
feudal/settings.py
env
static
deploy
runtest
__pycache__
How do I register a FEUDAL client?
---
- Go into the admin at `/backend/admin` and login
- Navigate to `Home > Backend > Users`
- Click "ADD USER" in top righthand corner
- Specify a username and a password
- The username and password need to be specified in the clients own config, see [here](https://git.scc.kit.edu/feudal/feudalClient)
- Click save
- Navigate to `Home > Backend > Sites`
- Click "ADD SITE" in top righthand corner
- For the client field select the client user you just created
- Specify a name, and optionally a description for the site
- Click save
Where are the logs?
---
- The logging depends on your configuration. Here are the default log locations:
- nginx logs to /var/log/nginx
- rabbitmq logs to /var/log/rabbitmq
- django logs to /home/feudal/logs (it creates multiple log files for different log levels)
- uwsgi logs to /home/feudal/logs/uwsgi.log
FEderated User Credential Deployment PortAL (FEUDAL)
====
- User interface: [Webpage](https://git.scc.kit.edu/feudal/feudalWebpage)
- Component at the sites: [Client](https://git.scc.kit.edu/feudal/feudalClient)
Installation (Debian 9.6)
----
- Add some dependency repositories
- Add repository for Erlang 21 (e.g. from [here](https://www.rabbitmq.com/install-debian.html#bintray-apt-repo-erlang))
- Add repository for node.js v10.x (e.g. from [nodesource](https://github.com/nodesource/distributions#debinstall))
- Add a user for the backend
- `useradd -m $user`
- login as $user
- `git clone https://git.scc.kit.edu/feudal/feudalBackend $backend`
- `cd $backend`
- `cp config.env.default config.env`
- Adjust the default values where needed
- $backend needs to the directory this repo is cloned into
- $domain is the domain of your host machine
- Run `./install_privileged` as root
- Run `./install` as $user
Installation
====
Starting the backend
----
- systemctl start nginx rabbitmq-server (as root)
- systemctl --user start uwsgi
Checking the status of the backend
----
- `./status`
Runtime Configuration
----
- For runtime configuration we use the django inbuilt admin interface.
- Default path of the django admin: `/backend/admin`
- The credentials for the admin were entered by you during the run of the `install` script
- Your OpenId Connect clients are configured in `Home > Backend > Oidc configs`
- The default redirect URI is: `/backend/auth/v1/callback`
- `scopes` is a list of strings (JSON)
- Users *and* FEUDAL Clients are managed in `Home > Backend > Users`
- You can manually add FEUDAL Clients
- You can specify admin users
- You need to configure the `sites`, which provide services to your users in `Home > Backend > Sites`
- Configure your RabbitMQ instance in `Home > Backend > Rabbit mq instances`
- apt install nginx rabbitmq-server uwsgi-plugin-python3 virtualenv default-libmysqlclient-dev gcc
- Notes: gcc is needed for the python package mysqlclient
- useradd -m feudal
- su feudal
- git clone git.scc.kit.edu/fum/fum_backend ~/backend
- Configure your database in ~/.my.cnf
- Configure django in ~/backend/django_backend/settings.py
- cd ~/backend
- ./generate-secret
- virtualenv -p /usr/bin/python3
- source env/bin/activate
- pip install Django django-cors-headers django-mysql django-polymorphic django-rest-polymorphic djangorestframework oic urllib3 mysqlclient pika
- Configure uwsgi in /etc/uwsgi/backend.ini
- Configure nginx in /etc/nginx/conf.d/feudal.conf
- su feudal
- cd ~/backend
- source env/bin/activate
- ./manage.py makemigrations backend
- ./manage.py migrate
- ./manage.py create superuser
"""
Django settings for django_backend project.
Generated by 'django-admin startproject' using Django 1.11.7.
For more information on this file, see
https://docs.djangoproject.com/en/1.11/topics/settings/
For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.11/ref/settings/
"""
import os
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = False
DEBUG_AUTH = False
DEBUG_PUBLISHING = False
DEBUG_CREDENTIALS = False
ALLOWED_HOSTS = [
'$domain',
]
AUTH_USER_MODEL = 'backend.User'
# cookie settings
......@@ -22,36 +22,40 @@ SESSION_COOKIE_AGE = 3600
SESSION_COOKIE_SECURE = True
SESSION_COOKIE_HTTPONLY = False
WSGI_APPLICATION = 'feudal.wsgi.application'
TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
CSRF_HEADER_NAME = 'HTTP_X_CSRFTOKEN'
CORS_ORIGIN_ALLOW_ALL = True
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
with open('.secret.key') as f:
with open('$secret') as f:
SECRET_KEY = f.read().strip()
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
ALLOWED_HOSTS = [
'hdf-portal.data.kit.edu',
]
# Application definition
ROOT_URLCONF = 'feudal.urls'
STATIC_URL = '/backend/static/'
STATIC_ROOT = '$static'
INSTALLED_APPS = [
'django.contrib.admin',
'django.contrib.auth',
'polymorphic',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'rest_framework',
'django_backend.backend',
'feudal.backend',
'corsheaders',
'django_mysql',
'django_nose',
]
MIDDLEWARE = [
......@@ -65,8 +69,6 @@ MIDDLEWARE = [
'django.middleware.clickjacking.XFrameOptionsMiddleware',
]
ROOT_URLCONF = 'django_backend.urls'
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
......@@ -83,43 +85,30 @@ TEMPLATES = [
},
]
WSGI_APPLICATION = 'django_backend.wsgi.application'
# Database
# https://docs.djangoproject.com/en/1.11/ref/settings/#databases
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.mysql',
'NAME': 'scc-hdfmysql0001_portal',
'USER': os.environ['DB_USER'],
'HOST': 'mysql2g.scc.kit.edu',
'PASSWORD': os.environ['DB_PASSWORD'],
'OPTIONS': {
'init_command': "SET innodb_strict_mode=1; SET sql_mode='STRICT_TRANS_TABLES'",
'charset': 'utf8mb4',
'read_default_file': os.path.expanduser('$mysql'),
'init_command': "SET innodb_strict_mode=1; SET sql_mode='STRICT_TRANS_TABLES';",
'charset': 'utf8mb4'
},
'TEST': {
'NAME': 'scc-hdfmysql0001_portal_test',
'CHARSET': 'utf8mb4',
'COLLATION': 'utf8mb4_unicode_ci',
'NAME': 'scc-hdfmysql0001_portal_dev_test',
}
}
}
CORS_ORIGIN_ALLOW_ALL = True
# AUTHENTICATION AND AUTHORIZATION
AUTHENTICATION_BACKENDS = [
'django_backend.backend.auth.v1.models.OIDCTokenAuthBackend',
'feudal.backend.auth.v1.OIDCTokenAuthBackend',
'django.contrib.auth.backends.ModelBackend',
]
REST_FRAMEWORK = {
'DEFAULT_AUTHENTICATION_CLASSES': [
'django_backend.backend.auth.v1.auth_class.CsrfExemptSessionAuthentication',
'rest_framework.authentication.SessionAuthentication',
'feudal.backend.auth.v1.OIDCTokenAuthHTTPBackend',
],
'DEFAULT_PERMISSION_CLASSES': [
'rest_framework.permissions.IsAuthenticated',
......@@ -127,8 +116,6 @@ REST_FRAMEWORK = {
}
# Password validation
# https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [
{
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
......@@ -144,27 +131,15 @@ AUTH_PASSWORD_VALIDATORS = [
},
]
# Internationalization
# https://docs.djangoproject.com/en/1.11/topics/i18n/
LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'UTC'
TIME_ZONE = 'Europe/Berlin'
USE_I18N = True
USE_L10N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.11/howto/static-files/
STATIC_URL = '/backend/static/'
STATIC_ROOT = 'static'
# LOGGING
LOGGING_ROOT = os.path.expanduser('$logs')
LOGGING = {
'version': 1,
'disable_existing_loggers': True,
......@@ -172,29 +147,38 @@ LOGGING = {
'standard': {
'format': '%(asctime)s - %(name)s - %(levelname)s - %(message)s',
},
'compact': {
'format': '%(levelname)s - %(message)s',
},
},
'handlers': {
'django': {
'class': 'logging.FileHandler',
'filename': './logs/django.log',
'filename': LOGGING_ROOT + '/django.log',
'formatter': 'standard',
},
'debug': {
'level': 'DEBUG',
'class': 'logging.FileHandler',
'filename': './logs/debug.log',
'filename': LOGGING_ROOT + '/debug.log',
'formatter': 'standard',
},
'compact-debug': {
'level': 'DEBUG',
'class': 'logging.FileHandler',
'filename': LOGGING_ROOT + '/compact-debug.log',
'formatter': 'compact',
},
'info': {
'level': 'INFO',
'class': 'logging.FileHandler',
'filename': './logs/info.log',
'filename': LOGGING_ROOT + '/info.log',
'formatter': 'standard',
},
'error': {
'level': 'ERROR',
'class': 'logging.FileHandler',
'filename': './logs/error.log',
'filename': LOGGING_ROOT + '/error.log',
'formatter': 'standard',
},
'console': {
......@@ -203,12 +187,12 @@ LOGGING = {
},
},
'loggers': {
'django_backend': {
'handlers': ['debug', 'info', 'error'],
'feudal': {
'handlers': ['debug', 'info', 'error', 'compact-debug'],
'level': 'DEBUG',
},
'django': {
'handlers': ['debug', 'info', 'error'],
'handlers': ['debug', 'info', 'error', 'compact-debug'],
'level': 'INFO',
},
},
......
[client]
user = $db_user
password = $db_password
database = $db_name
host = $db_host
default-character-set = utf8mb4
init-command = "SET innodb_strict_mode=1; SET sql_mode='STRICT_TRANS_TABLES'"
......@@ -5,11 +5,11 @@ map $http_upgrade $connection_upgrade {
}
upstream django {
server unix://home/feudal/backend/feudal.sock;
server unix://$uwsgi_socket;
}
upstream websocket {
server 127.0.0.1:15674;
server 127.0.0.1:$port_websocket;
}
server {
......@@ -23,42 +23,68 @@ server {
listen 443 ssl http2 default_server;
listen [::]:443 ssl http2 default_server;
server_name hdf-portal.data.kit.edu;
server_name $domain;
charset utf-8;
client_max_body_size 75M;
gzip on;
gzip_types text/plain application/javascript;
gzip_proxied no-cache no-store private expired auth;
gzip_min_length 1000;
ssl on;
ssl_certificate /etc/ssl/hdf-portal.data.kit.edu/chain.pem;
ssl_trusted_certificate /etc/ssl/hdf-portal.data.kit.edu/fullchain.pem;
ssl_certificate_key /etc/ssl/hdf-portal.data.kit.edu/key.pem;
ssl_certificate $ssl_chain;
ssl_trusted_certificate $ssl_fullchain;
ssl_certificate_key $ssl_key;
ssl_dhparam $dhparam;
ssl_session_cache shared:SSL:20m;
ssl_session_timeout 180m;
ssl_prefer_server_ciphers on;
ssl_ciphers ECDH+AESGCM:ECDH+AES256:ECDH+AES128:DHE+AES128:!ADH:!AECDH:!MD5;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_dhparam /etc/nginx/cert/dhparam.pem;
ssl_protocols TLSv1.1 TLSv1.2;
ssl_stapling on;
ssl_stapling_verify on;
resolver 141.3.175.65 141.3.175.66;
add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
root /home/feudal/webpage/dist;
root $dist;
index index.html;
# Cache webpage assets
location /assets {
alias $dist/assets/;
expires 30d;
add_header Pragma public;
add_header Cache-Control "public";
}
location /backend/static {
alias /home/feudal/backend/static;
# Cache static assets of the django admin
location ^~ /backend/static/ {
alias $static/;
expires 30d;
add_header Pragma public;
add_header Cache-Control "public";
}
# Shortened path for the user rest interface
location /rest {
rewrite ^/rest/(.*)$ /backend/user-api/$1;
}
# Calls to the backend are handled by uwsgi (see upstream django)
location /backend {
uwsgi_pass django;
include /home/feudal/backend/example-config/uwsgi_params;
include $config/uwsgi_params;
}
# The webpage is placed at the root (including its index.html)
location /frontend {
try_files $uri $uri/ /index.html;
}
# Handle websocket
location /ws {
proxy_pass http://websocket/ws;
proxy_http_version 1.1;
......@@ -66,8 +92,19 @@ server {
proxy_set_header Connection $connection_upgrade;
}
# Redirect to direct login when we have the 'idp' parameter
location / {
if ($arg_idp) {
return 301 /backend/auth/v1/request$is_args$args;
}
rewrite "^$" /frontend;
}
# Cache images
location ~* \.(?:ico|css|js|gif|jpe?g|png)$ {
expires 30d;
add_header Pragma public;
add_header Cache-Control "public";
}
}
### listeners
# amqps
listeners.ssl.default = 5671
# websocket
web_stomp.port = $port_websocket
# internal stomp port
stomp.listeners.tcp.1 = 61613
### auth backends
# internal backend -> localhost authentication
auth_backends.1 = internal
# http ackend -> delegated authentication of feudal clients at the feudal backend
auth_backends.2 = http
# external authentication endpoints at the feudal backend
# replace domain with your domain
auth_http.http_method = post
auth_http.user_path = https://$domain:443/backend/auth/v1/client/user
auth_http.vhost_path = https://$domain:443/backend/auth/v1/client/vhost
auth_http.resource_path = https://$domain:443/backend/auth/v1/client/resource
auth_http.topic_path = https://$domain:443/backend/auth/v1/client/topic
# ssl setup (needed for amqp)
ssl_options.cacertfile = $ssl_chain
ssl_options.certfile = $ssl_cert
ssl_options.keyfile = $ssl_key
ssl_options.verify = verify_none
ssl_options.fail_if_no_peer_cert = false
# stomp
stomp.default_user = guest
stomp.default_pass = guest
......@@ -2,20 +2,25 @@
# Django-related settings
# the base directory (full path)
chdir = /home/feudal/backend
chdir = $backend
# Django's wsgi file
module = feudal.wsgi
# the virtualenv (full path)
home = /home/feudal/backend/env
home = $venv
plugins = python35
# process-related settings
# master
master = true
# maximum number of worker processes
processes = 1
# the socket (use the full path to be safe
socket = /home/feudal/backend/feudal.sock
# the socket
socket = $uwsgi_socket
# ... with appropriate permissions - may be needed
chmod-socket = 666
# clear environment on exit
vacuum = true
logto = $logs/uwsgi.log
[Unit]
Description=uWSGI Emperor
After=syslog.target
[Service]
Type=notify
ExecStart=/usr/bin/uwsgi --ini $uwsgi_ini
RuntimeDirectory=uwsgi
KillSignal=SIGQUIT
StandardError=syslog
Restart=on-failure
[Install]
WantedBy=default.target
# vim: set filetype=bash foldmethod=marker :
# the domain / host of this machine
export domain=hdf-portal-dev.data.kit.edu
# the user which runs the backend
export user=feudal
export root=/home/$user
# dir for the config files
export config=$root/config
# the directory for the logs (unprivileged)
export logs=$root/logs
systemd=/home/$user/.local/share/systemd/user
# {{{ uwsgi
export uwsgi_ini=$config/uwsgi.ini
export uwsgi_socket=$root/feudal.sock
# }}}
# {{{ backend
# python version (currently only tested with 3.5)
export python=$(which python3.5)
# the directory into which you cloned https://git.scc.kit.edu/feudal/feudalBackend
export backend=$root/feudalBackend
export static=$backend/static
export venv=$backend/env
export secret=$config/django.key
export mysql=$config/mysql.cnf
export port_websocket=15674
# }}}
# {{{ webpage
# the directory for https://git.scc.kit.edu/feudal/feudalWebpage
export webpage=$root/feudalWebpage
export dist=$webpage/dist
# }}}
# {{{ ssl certificate
ssl=/etc/ssl/$domain
export ssl_chain=$ssl/chain.pem
export ssl_cert=$ssl/cert.pem
export ssl_fullchain=$ssl/fullchain.pem
export ssl_key=$ssl/key.pem
export dhparam=/etc/nginx/cert/dhparam.pem
# }}}
# {{{ db
export db_user=
export db_password=
export db_host=
export db_name=
# }}}
from django.contrib import admin
from django.contrib.auth.admin import UserAdmin
from django.contrib.auth.models import Group
from . import models, broker_models
from .auth.v1.models import OIDCConfig
class TypeFilter(admin.SimpleListFilter):
title = 'Type'
parameter_name = 'user_type'
def lookups(self, request, model_admin):
return models.User.TYPE_CHOICES
def queryset(self, request, queryset):
if self.value():
return queryset.filter(user_type=self.value())
return queryset
class ClientAdmin(UserAdmin):
list_filter = (TypeFilter,)
admin.site.register(OIDCConfig)
admin.site.register(broker_models.RabbitMQInstance)
admin.site.register(models.User, ClientAdmin)
admin.site.unregister(Group)
admin.site.register(Group)
admin.site.register(models.Site)
admin.site.register(models.Service)
admin.site.register(models.SSHPublicKey)
admin.site.register(models.Deployment)
admin.site.register(models.DeploymentState)
admin.site.register(models.DeploymentStateItem)
from rest_framework.authentication import SessionAuthentication
class CsrfExemptSessionAuthentication(SessionAuthentication):
def enforce_csrf(self, request):
return # To not perform the csrf check previously happening
# we don't need to deserialize, so we do not implement the abstract methods
# pylint: disable=abstract-method
from rest_framework import serializers
from .models import OIDCConfig
class OIDCConfigSerializer(serializers.ModelSerializer):
class Meta:
model = OIDCConfig
fields = ['name', 'id']
class AuthInfoSerializer(serializers.Serializer):
idps = OIDCConfigSerializer(many=True)
default = serializers.IntegerField()
from django.conf.urls import url
from django.views.decorators.csrf import csrf_exempt
from . import views, client_views
URLPATTERNS = [
url(r'^info', views.AuthInfo.as_view()),
url(r'^request', views.Auth.as_view()),
url(r'^callback', views.AuthCallback.as_view()),
url(r'^logout', views.LogoutView.as_view()),
url(r'^client/user', csrf_exempt(client_views.user_endpoint)),
url(r'^client/vhost', csrf_exempt(client_views.vhost_endpoint)),
url(r'^client/resource', csrf_exempt(client_views.resource_endpoint)),
url(r'^client/topic', csrf_exempt(client_views.topic_endpoint)),
]
import json
import logging
from django.contrib.auth import authenticate, login, logout
from django.shortcuts import redirect
from django.views import View
from rest_framework import generics, views
from rest_framework.permissions import AllowAny
from rest_framework.response import Response
from oic import rndstr
from oic.oic.message import AuthorizationResponse
from .models import OIDCConfig, default_idp
from .serializers import AuthInfoSerializer
from . import utils
from ...frontend.views import state_view_data
LOGGER = logging.getLogger(__name__)
IDP_COOKIE_NAME = 'idp_id'
def idp_id_from_request(request):
idp_id = request.COOKIES.get(IDP_COOKIE_NAME, None)
if idp_id is not None:
return idp_id
LOGGER.info('idp_id cookie was not set on request %s', request)
return default_idp().id
class AuthException(Exception):
pass
class Auth(View):
def get(self, request):
try:
state = rndstr()