Commit e6de58db authored by Lukas Burgey's avatar Lukas Burgey

Merge branch 'dev'

parents fbee6a81 b343b80b
.secret.key .secret.key
feudal/settings.py
env env
static static
deploy deploy
runtest runtest
__pycache__
How do I register a FEUDAL client?
---
- Go into the admin at `/backend/admin` and login
- Navigate to `Home > Backend > Users`
- Click "ADD USER" in top righthand corner
- Specify a username and a password
- The username and password need to be specified in the clients own config, see [here](https://git.scc.kit.edu/feudal/feudalClient)
- Click save
- Navigate to `Home > Backend > Sites`
- Click "ADD SITE" in top righthand corner
- For the client field select the client user you just created
- Specify a name, and optionally a description for the site
- Click save
Where are the logs?
---
- The logging depends on your configuration. Here are the default log locations:
- nginx logs to /var/log/nginx
- rabbitmq logs to /var/log/rabbitmq
- django logs to /home/feudal/logs (it creates multiple log files for different log levels)
- uwsgi logs to /home/feudal/logs/uwsgi.log
FEderated User Credential Deployment PortAL (FEUDAL)
====
- User interface: [Webpage](https://git.scc.kit.edu/feudal/feudalWebpage)
- Component at the sites: [Client](https://git.scc.kit.edu/feudal/feudalClient)
Installation (Debian 9.6)
----
- Add some dependency repositories
- Add repository for Erlang 21 (e.g. from [here](https://www.rabbitmq.com/install-debian.html#bintray-apt-repo-erlang))
- Add repository for node.js v10.x (e.g. from [nodesource](https://github.com/nodesource/distributions#debinstall))
- Add a user for the backend
- `useradd -m $user`
- login as $user
- `git clone https://git.scc.kit.edu/feudal/feudalBackend $backend`
- `cd $backend`
- `cp config.env.default config.env`
- Adjust the default values where needed
- $backend needs to the directory this repo is cloned into
- $domain is the domain of your host machine
- Run `./install_privileged` as root
- Run `./install` as $user
Installation Starting the backend
==== ----
- systemctl start nginx rabbitmq-server (as root)
- systemctl --user start uwsgi
Checking the status of the backend
----
- `./status`
Runtime Configuration
----
- For runtime configuration we use the django inbuilt admin interface.
- Default path of the django admin: `/backend/admin`
- The credentials for the admin were entered by you during the run of the `install` script
- Your OpenId Connect clients are configured in `Home > Backend > Oidc configs`
- The default redirect URI is: `/backend/auth/v1/callback`
- `scopes` is a list of strings (JSON)
- Users *and* FEUDAL Clients are managed in `Home > Backend > Users`
- You can manually add FEUDAL Clients
- You can specify admin users
- You need to configure the `sites`, which provide services to your users in `Home > Backend > Sites`
- Configure your RabbitMQ instance in `Home > Backend > Rabbit mq instances`
- apt install nginx rabbitmq-server uwsgi-plugin-python3 virtualenv default-libmysqlclient-dev gcc
- Notes: gcc is needed for the python package mysqlclient
- useradd -m feudal
- su feudal
- git clone git.scc.kit.edu/fum/fum_backend ~/backend
- Configure your database in ~/.my.cnf
- Configure django in ~/backend/django_backend/settings.py
- cd ~/backend
- ./generate-secret
- virtualenv -p /usr/bin/python3
- source env/bin/activate
- pip install Django django-cors-headers django-mysql django-polymorphic django-rest-polymorphic djangorestframework oic urllib3 mysqlclient pika
- Configure uwsgi in /etc/uwsgi/backend.ini
- Configure nginx in /etc/nginx/conf.d/feudal.conf
- su feudal
- cd ~/backend
- source env/bin/activate
- ./manage.py makemigrations backend
- ./manage.py migrate
- ./manage.py create superuser
"""
Django settings for django_backend project.
Generated by 'django-admin startproject' using Django 1.11.7.
For more information on this file, see
https://docs.djangoproject.com/en/1.11/topics/settings/
For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.11/ref/settings/
"""
import os import os
# Build paths inside the project like this: os.path.join(BASE_DIR, ...) # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = False
DEBUG_AUTH = False
DEBUG_PUBLISHING = False
DEBUG_CREDENTIALS = False
ALLOWED_HOSTS = [
'$domain',
]
AUTH_USER_MODEL = 'backend.User' AUTH_USER_MODEL = 'backend.User'
# cookie settings # cookie settings
...@@ -22,36 +22,40 @@ SESSION_COOKIE_AGE = 3600 ...@@ -22,36 +22,40 @@ SESSION_COOKIE_AGE = 3600
SESSION_COOKIE_SECURE = True SESSION_COOKIE_SECURE = True
SESSION_COOKIE_HTTPONLY = False SESSION_COOKIE_HTTPONLY = False
WSGI_APPLICATION = 'feudal.wsgi.application'
TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
CSRF_HEADER_NAME = 'HTTP_X_CSRFTOKEN' CSRF_HEADER_NAME = 'HTTP_X_CSRFTOKEN'
CORS_ORIGIN_ALLOW_ALL = True
# Quick-start development settings - unsuitable for production # Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/ # See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret! # SECURITY WARNING: keep the secret key used in production secret!
with open('.secret.key') as f: with open('$secret') as f:
SECRET_KEY = f.read().strip() SECRET_KEY = f.read().strip()
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
ALLOWED_HOSTS = [
'hdf-portal.data.kit.edu',
]
# Application definition # Application definition
ROOT_URLCONF = 'feudal.urls'
STATIC_URL = '/backend/static/'
STATIC_ROOT = '$static'
INSTALLED_APPS = [ INSTALLED_APPS = [
'django.contrib.admin', 'django.contrib.admin',
'django.contrib.auth', 'django.contrib.auth',
'polymorphic',
'django.contrib.contenttypes', 'django.contrib.contenttypes',
'django.contrib.sessions', 'django.contrib.sessions',
'django.contrib.messages', 'django.contrib.messages',
'django.contrib.staticfiles', 'django.contrib.staticfiles',
'rest_framework', 'rest_framework',
'django_backend.backend', 'feudal.backend',
'corsheaders', 'corsheaders',
'django_mysql', 'django_mysql',
'django_nose',
] ]
MIDDLEWARE = [ MIDDLEWARE = [
...@@ -65,8 +69,6 @@ MIDDLEWARE = [ ...@@ -65,8 +69,6 @@ MIDDLEWARE = [
'django.middleware.clickjacking.XFrameOptionsMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware',
] ]
ROOT_URLCONF = 'django_backend.urls'
TEMPLATES = [ TEMPLATES = [
{ {
'BACKEND': 'django.template.backends.django.DjangoTemplates', 'BACKEND': 'django.template.backends.django.DjangoTemplates',
...@@ -83,43 +85,30 @@ TEMPLATES = [ ...@@ -83,43 +85,30 @@ TEMPLATES = [
}, },
] ]
WSGI_APPLICATION = 'django_backend.wsgi.application'
# Database
# https://docs.djangoproject.com/en/1.11/ref/settings/#databases
DATABASES = { DATABASES = {
'default': { 'default': {
'ENGINE': 'django.db.backends.mysql', 'ENGINE': 'django.db.backends.mysql',
'NAME': 'scc-hdfmysql0001_portal',
'USER': os.environ['DB_USER'],
'HOST': 'mysql2g.scc.kit.edu',
'PASSWORD': os.environ['DB_PASSWORD'],
'OPTIONS': { 'OPTIONS': {
'init_command': "SET innodb_strict_mode=1; SET sql_mode='STRICT_TRANS_TABLES'", 'read_default_file': os.path.expanduser('$mysql'),
'charset': 'utf8mb4', 'init_command': "SET innodb_strict_mode=1; SET sql_mode='STRICT_TRANS_TABLES';",
'charset': 'utf8mb4'
}, },
'TEST': { 'TEST': {
'NAME': 'scc-hdfmysql0001_portal_test', 'NAME': 'scc-hdfmysql0001_portal_dev_test',
'CHARSET': 'utf8mb4',
'COLLATION': 'utf8mb4_unicode_ci',
} }
} }
} }
CORS_ORIGIN_ALLOW_ALL = True
# AUTHENTICATION AND AUTHORIZATION # AUTHENTICATION AND AUTHORIZATION
AUTHENTICATION_BACKENDS = [ AUTHENTICATION_BACKENDS = [
'django_backend.backend.auth.v1.models.OIDCTokenAuthBackend', 'feudal.backend.auth.v1.OIDCTokenAuthBackend',
'django.contrib.auth.backends.ModelBackend', 'django.contrib.auth.backends.ModelBackend',
] ]
REST_FRAMEWORK = { REST_FRAMEWORK = {
'DEFAULT_AUTHENTICATION_CLASSES': [ 'DEFAULT_AUTHENTICATION_CLASSES': [
'django_backend.backend.auth.v1.auth_class.CsrfExemptSessionAuthentication', 'rest_framework.authentication.SessionAuthentication',
'feudal.backend.auth.v1.OIDCTokenAuthHTTPBackend',
], ],
'DEFAULT_PERMISSION_CLASSES': [ 'DEFAULT_PERMISSION_CLASSES': [
'rest_framework.permissions.IsAuthenticated', 'rest_framework.permissions.IsAuthenticated',
...@@ -127,8 +116,6 @@ REST_FRAMEWORK = { ...@@ -127,8 +116,6 @@ REST_FRAMEWORK = {
} }
# Password validation # Password validation
# https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [ AUTH_PASSWORD_VALIDATORS = [
{ {
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
...@@ -144,27 +131,15 @@ AUTH_PASSWORD_VALIDATORS = [ ...@@ -144,27 +131,15 @@ AUTH_PASSWORD_VALIDATORS = [
}, },
] ]
# Internationalization # Internationalization
# https://docs.djangoproject.com/en/1.11/topics/i18n/
LANGUAGE_CODE = 'en-us' LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'Europe/Berlin'
TIME_ZONE = 'UTC'
USE_I18N = True USE_I18N = True
USE_L10N = True USE_L10N = True
USE_TZ = True USE_TZ = True
# LOGGING
# Static files (CSS, JavaScript, Images) LOGGING_ROOT = os.path.expanduser('$logs')
# https://docs.djangoproject.com/en/1.11/howto/static-files/
STATIC_URL = '/backend/static/'
STATIC_ROOT = 'static'
LOGGING = { LOGGING = {
'version': 1, 'version': 1,
'disable_existing_loggers': True, 'disable_existing_loggers': True,
...@@ -172,29 +147,38 @@ LOGGING = { ...@@ -172,29 +147,38 @@ LOGGING = {
'standard': { 'standard': {
'format': '%(asctime)s - %(name)s - %(levelname)s - %(message)s', 'format': '%(asctime)s - %(name)s - %(levelname)s - %(message)s',
}, },
'compact': {
'format': '%(levelname)s - %(message)s',
},
}, },
'handlers': { 'handlers': {
'django': { 'django': {
'class': 'logging.FileHandler', 'class': 'logging.FileHandler',
'filename': './logs/django.log', 'filename': LOGGING_ROOT + '/django.log',
'formatter': 'standard', 'formatter': 'standard',
}, },
'debug': { 'debug': {
'level': 'DEBUG', 'level': 'DEBUG',
'class': 'logging.FileHandler', 'class': 'logging.FileHandler',
'filename': './logs/debug.log', 'filename': LOGGING_ROOT + '/debug.log',
'formatter': 'standard', 'formatter': 'standard',
}, },
'compact-debug': {
'level': 'DEBUG',
'class': 'logging.FileHandler',
'filename': LOGGING_ROOT + '/compact-debug.log',
'formatter': 'compact',
},
'info': { 'info': {
'level': 'INFO', 'level': 'INFO',
'class': 'logging.FileHandler', 'class': 'logging.FileHandler',
'filename': './logs/info.log', 'filename': LOGGING_ROOT + '/info.log',
'formatter': 'standard', 'formatter': 'standard',
}, },
'error': { 'error': {
'level': 'ERROR', 'level': 'ERROR',
'class': 'logging.FileHandler', 'class': 'logging.FileHandler',
'filename': './logs/error.log', 'filename': LOGGING_ROOT + '/error.log',
'formatter': 'standard', 'formatter': 'standard',
}, },
'console': { 'console': {
...@@ -203,12 +187,12 @@ LOGGING = { ...@@ -203,12 +187,12 @@ LOGGING = {
}, },
}, },
'loggers': { 'loggers': {
'django_backend': { 'feudal': {
'handlers': ['debug', 'info', 'error'], 'handlers': ['debug', 'info', 'error', 'compact-debug'],
'level': 'DEBUG', 'level': 'DEBUG',
}, },
'django': { 'django': {
'handlers': ['debug', 'info', 'error'], 'handlers': ['debug', 'info', 'error', 'compact-debug'],
'level': 'INFO', 'level': 'INFO',
}, },
}, },
......
[client]
user = $db_user
password = $db_password
database = $db_name
host = $db_host
default-character-set = utf8mb4
init-command = "SET innodb_strict_mode=1; SET sql_mode='STRICT_TRANS_TABLES'"
...@@ -5,11 +5,11 @@ map $http_upgrade $connection_upgrade { ...@@ -5,11 +5,11 @@ map $http_upgrade $connection_upgrade {
} }
upstream django { upstream django {
server unix://home/feudal/backend/feudal.sock; server unix://$uwsgi_socket;
} }
upstream websocket { upstream websocket {
server 127.0.0.1:15674; server 127.0.0.1:$port_websocket;
} }
server { server {
...@@ -23,42 +23,68 @@ server { ...@@ -23,42 +23,68 @@ server {
listen 443 ssl http2 default_server; listen 443 ssl http2 default_server;
listen [::]:443 ssl http2 default_server; listen [::]:443 ssl http2 default_server;
server_name hdf-portal.data.kit.edu; server_name $domain;
charset utf-8; charset utf-8;
client_max_body_size 75M; client_max_body_size 75M;
gzip on;
gzip_types text/plain application/javascript;
gzip_proxied no-cache no-store private expired auth;
gzip_min_length 1000;
ssl on; ssl on;
ssl_certificate /etc/ssl/hdf-portal.data.kit.edu/chain.pem; ssl_certificate $ssl_chain;
ssl_trusted_certificate /etc/ssl/hdf-portal.data.kit.edu/fullchain.pem; ssl_trusted_certificate $ssl_fullchain;
ssl_certificate_key /etc/ssl/hdf-portal.data.kit.edu/key.pem; ssl_certificate_key $ssl_key;
ssl_dhparam $dhparam;
ssl_session_cache shared:SSL:20m; ssl_session_cache shared:SSL:20m;
ssl_session_timeout 180m; ssl_session_timeout 180m;
ssl_prefer_server_ciphers on; ssl_prefer_server_ciphers on;
ssl_ciphers ECDH+AESGCM:ECDH+AES256:ECDH+AES128:DHE+AES128:!ADH:!AECDH:!MD5; ssl_ciphers ECDH+AESGCM:ECDH+AES256:ECDH+AES128:DHE+AES128:!ADH:!AECDH:!MD5;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2; ssl_protocols TLSv1.1 TLSv1.2;
ssl_dhparam /etc/nginx/cert/dhparam.pem;
ssl_stapling on; ssl_stapling on;
ssl_stapling_verify on; ssl_stapling_verify on;
resolver 141.3.175.65 141.3.175.66; resolver 141.3.175.65 141.3.175.66;
add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
root /home/feudal/webpage/dist; root $dist;
index index.html; index index.html;
# Cache webpage assets
location /assets {
alias $dist/assets/;
expires 30d;
add_header Pragma public;
add_header Cache-Control "public";
}
location /backend/static { # Cache static assets of the django admin
alias /home/feudal/backend/static; location ^~ /backend/static/ {
alias $static/;
expires 30d;
add_header Pragma public;
add_header Cache-Control "public";
} }
# Shortened path for the user rest interface
location /rest {
rewrite ^/rest/(.*)$ /backend/user-api/$1;
}
# Calls to the backend are handled by uwsgi (see upstream django)
location /backend { location /backend {
uwsgi_pass django; uwsgi_pass django;
include /home/feudal/backend/example-config/uwsgi_params; include $config/uwsgi_params;
} }
# The webpage is placed at the root (including its index.html)
location /frontend { location /frontend {
try_files $uri $uri/ /index.html; try_files $uri $uri/ /index.html;
} }
# Handle websocket
location /ws { location /ws {
proxy_pass http://websocket/ws; proxy_pass http://websocket/ws;
proxy_http_version 1.1; proxy_http_version 1.1;
...@@ -66,8 +92,19 @@ server { ...@@ -66,8 +92,19 @@ server {
proxy_set_header Connection $connection_upgrade; proxy_set_header Connection $connection_upgrade;
} }
# Redirect to direct login when we have the 'idp' parameter
location / { location / {
if ($arg_idp) {
return 301 /backend/auth/v1/request$is_args$args;
}
rewrite "^$" /frontend; rewrite "^$" /frontend;
} }
# Cache images
location ~* \.(?:ico|css|js|gif|jpe?g|png)$ {
expires 30d;
add_header Pragma public;
add_header Cache-Control "public";
}
} }
### listeners
# amqps
listeners.ssl.default = 5671
# websocket
web_stomp.port = $port_websocket
# internal stomp port
stomp.listeners.tcp.1 = 61613
### auth backends
# internal backend -> localhost authentication
auth_backends.1 = internal
# http ackend -> delegated authentication of feudal clients at the feudal backend
auth_backends.2 = http
# external authentication endpoints at the feudal backend
# replace domain with your domain
auth_http.http_method = post
auth_http.user_path = https://$domain:443/backend/auth/v1/client/user
auth_http.vhost_path = https://$domain:443/backend/auth/v1/client/vhost
auth_http.resource_path = https://$domain:443/backend/auth/v1/client/resource
auth_http.topic_path = https://$domain:443/backend/auth/v1/client/topic
# ssl setup (needed for amqp)
ssl_options.cacertfile = $ssl_chain
ssl_options.certfile = $ssl_cert
ssl_options.keyfile = $ssl_key
ssl_options.verify = verify_none
ssl_options.fail_if_no_peer_cert = false
# stomp
stomp.default_user = guest
stomp.default_pass = guest
...@@ -2,20 +2,25 @@ ...@@ -2,20 +2,25 @@
# Django-related settings # Django-related settings
# the base directory (full path) # the base directory (full path)
chdir = /home/feudal/backend chdir = $backend
# Django's wsgi file # Django's wsgi file
module = feudal.wsgi module = feudal.wsgi
# the virtualenv (full path) # the virtualenv (full path)
home = /home/feudal/backend/env home = $venv
plugins = python35
# process-related settings # process-related settings
# master # master
master = true master = true
# maximum number of worker processes # maximum number of worker processes
processes = 1 processes = 1
# the socket (use the full path to be safe # the socket
socket = /home/feudal/backend/feudal.sock socket = $uwsgi_socket
# ... with appropriate permissions - may be needed # ... with appropriate permissions - may be needed
chmod-socket = 666 chmod-socket = 666
# clear environment on exit # clear environment on exit
vacuum = true vacuum = true
logto = $logs/uwsgi.log
[Unit]
Description=uWSGI Emperor
After=syslog.target
[Service]
Type=notify
ExecStart=/usr/bin/uwsgi --ini $uwsgi_ini
RuntimeDirectory=uwsgi
KillSignal=SIGQUIT
StandardError=syslog
Restart=on-failure
[Install]
WantedBy=default.target
# vim: set filetype=bash foldmethod=marker :
# the domain / host of this machine
export domain=hdf-portal-dev.data.kit.edu
# the user which runs the backend
export user=feudal
export root=/home/$user
# dir for the config files
export config=$root/config
# the directory for the logs (unprivileged)
export logs=$root/logs
systemd=/home/$user/.local/share/systemd/user
# {{{ uwsgi
export uwsgi_ini=$config/uwsgi.ini
export uwsgi_socket=$root/feudal.sock
# }}}
# {{{ backend
# python version (currently only tested with 3.5)
export python=$(which python3.5)
# the directory into which you cloned https://git.scc.kit.edu/feudal/feudalBackend
export backend=$root/feudalBackend
export static=$backend/static
export venv=$backend/env
export secret=$config/django.key
export mysql=$config/mysql.cnf
export port_websocket=15674
# }}}
# {{{ webpage
# the directory for https://git.scc.kit.edu/feudal/feudalWebpage
export webpage=$root/feudalWebpage
export dist=$webpage/dist
# }}}