Commit b343b80b authored by Lukas Burgey's avatar Lukas Burgey

Rework installation process

parent f8c0b4f3
......@@ -3,43 +3,32 @@ FEderated User Credential Deployment PortAL (FEUDAL)
- User interface: [Webpage](https://git.scc.kit.edu/feudal/feudalWebpage)
- Component at the sites: [Client](https://git.scc.kit.edu/feudal/feudalClient)
Installation
Installation (Debian 9.6)
----
- apt install nginx uwsgi-plugin-python3 virtualenv default-libmysqlclient-dev gcc
- Notes: gcc is needed for the python package mysqlclient
- Install rabbitmq 3.7.7 (e.g. from: [here](https://www.rabbitmq.com/install-debian.html#bintray-apt-repo-erlang))
- Add some dependency repositories
- Add repository for Erlang 21 (e.g. from [here](https://www.rabbitmq.com/install-debian.html#bintray-apt-repo-erlang))
- Add repository for node.js v10.x (e.g. from [nodesource](https://github.com/nodesource/distributions#debinstall))
- Add a user for the backend
- useradd -m feudal
- su feudal
- git clone git.scc.kit.edu/feudal/feudalBackend ~/feudalBackend
- cd ~/feudalBackend
- ./install
Configuration
- `useradd -m $user`
- login as $user
- `git clone https://git.scc.kit.edu/feudal/feudalBackend $backend`
- `cd $backend`
- `cp config.env.default config.env`
- Adjust the default values where needed
- $backend needs to the directory this repo is cloned into
- $domain is the domain of your host machine
- Run `./install_privileged` as root
- Run `./install` as $user
Starting the backend
----
- systemctl start nginx rabbitmq-server (as root)
- systemctl --user start uwsgi
- Please inspect the [example configuration](https://git.scc.kit.edu/feudal/feudalBackend/tree/master/example-config)
- Configure your database in ~/.my.cnf
- Django / UWSGI configuration is done in ~/config
- Configure django in ~/config/django_settings.py
- Configure uwsgi using ~/config/uwsgi.ini
- Enable the uwsgi service using systemd
- systemctl --user --now enable uwsgi
- Configure nginx in /etc/nginx/conf.d/feudal.conf
- Copy the dhparam.pem file to /etc/nginx/cert
- Enable nginx service using systemd
- systemctl --now enable nginx
- Configure rabbitmq in /etc/rabbitmq/rabbitmq.conf
- Enable rabbitmq plugins:
- rabbitmq-plugins enable rabbitmq_web_stomp rabbitmq_stomp rabbitmq_auth_backend_http
- Enable rabbitmq using systemd
- systemctl --now enable rabbitmq-server
Checking the status of the backend
----
- `./status`
Runtime Configuration
......
......@@ -5,14 +5,14 @@ import os
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
DEBUG = False
DEBUG_AUTH = False
DEBUG_PUBLISHING = False
DEBUG_CREDENTIALS = False
ALLOWED_HOSTS = [
'hdf-portal-dev.data.kit.edu',
'$domain',
]
AUTH_USER_MODEL = 'backend.User'
......@@ -22,21 +22,26 @@ SESSION_COOKIE_AGE = 3600
SESSION_COOKIE_SECURE = True
SESSION_COOKIE_HTTPONLY = False
WSGI_APPLICATION = 'feudal.wsgi.application'
TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
CSRF_HEADER_NAME = 'HTTP_X_CSRFTOKEN'
CORS_ORIGIN_ALLOW_ALL = True
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
with open('.secret.key') as f:
with open('$secret') as f:
SECRET_KEY = f.read().strip()
# Application definition
ROOT_URLCONF = 'feudal.urls'
STATIC_URL = '/backend/static/'
STATIC_ROOT = 'static'
STATIC_ROOT = '$static'
INSTALLED_APPS = [
'django.contrib.admin',
......@@ -64,7 +69,6 @@ MIDDLEWARE = [
'django.middleware.clickjacking.XFrameOptionsMiddleware',
]
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
......@@ -81,16 +85,11 @@ TEMPLATES = [
},
]
WSGI_APPLICATION = 'feudal.wsgi.application'
# Database
# https://docs.djangoproject.com/en/1.11/ref/settings/#databases
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.mysql',
'OPTIONS': {
'read_default_file': os.path.expanduser('~/.my.cnf'),
'read_default_file': os.path.expanduser('$mysql'),
'init_command': "SET innodb_strict_mode=1; SET sql_mode='STRICT_TRANS_TABLES';",
'charset': 'utf8mb4'
},
......@@ -100,9 +99,6 @@ DATABASES = {
}
}
CORS_ORIGIN_ALLOW_ALL = True
# AUTHENTICATION AND AUTHORIZATION
AUTHENTICATION_BACKENDS = [
'feudal.backend.auth.v1.OIDCTokenAuthBackend',
......@@ -111,7 +107,7 @@ AUTHENTICATION_BACKENDS = [
REST_FRAMEWORK = {
'DEFAULT_AUTHENTICATION_CLASSES': [
'feudal.backend.auth.v1.CsrfExemptSessionAuthentication',
'rest_framework.authentication.SessionAuthentication',
'feudal.backend.auth.v1.OIDCTokenAuthHTTPBackend',
],
'DEFAULT_PERMISSION_CLASSES': [
......@@ -120,8 +116,6 @@ REST_FRAMEWORK = {
}
# Password validation
# https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [
{
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
......@@ -137,23 +131,15 @@ AUTH_PASSWORD_VALIDATORS = [
},
]
# Internationalization
# https://docs.djangoproject.com/en/1.11/topics/i18n/
LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'Europe/Berlin'
USE_I18N = True
USE_L10N = True
USE_TZ = True
# LOGGING
LOGGING_ROOT = os.path.expanduser('~/logs')
LOGGING_ROOT = os.path.expanduser('$logs')
LOGGING = {
'version': 1,
'disable_existing_loggers': True,
......@@ -211,5 +197,3 @@ LOGGING = {
},
},
}
TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
[client]
user = $db_user
password = $db_password
database = $db_name
host = $db_host
default-character-set = utf8mb4
init-command = "SET innodb_strict_mode=1; SET sql_mode='STRICT_TRANS_TABLES'"
......@@ -5,11 +5,11 @@ map $http_upgrade $connection_upgrade {
}
upstream django {
server unix://home/feudal/feudalBackend/feudal.sock;
server unix://$uwsgi_socket;
}
upstream websocket {
server 127.0.0.1:15674;
server 127.0.0.1:$port_websocket;
}
server {
......@@ -23,7 +23,7 @@ server {
listen 443 ssl http2 default_server;
listen [::]:443 ssl http2 default_server;
server_name hdf-portal-dev.data.kit.edu;
server_name $domain;
charset utf-8;
client_max_body_size 75M;
......@@ -33,10 +33,10 @@ server {
gzip_min_length 1000;
ssl on;
ssl_certificate /etc/ssl/hdf-portal-dev.data.kit.edu/chain.pem;
ssl_trusted_certificate /etc/ssl/hdf-portal-dev.data.kit.edu/fullchain.pem;
ssl_certificate_key /etc/ssl/hdf-portal-dev.data.kit.edu/key.pem;
ssl_dhparam /etc/nginx/cert/dhparam.pem;
ssl_certificate $ssl_chain;
ssl_trusted_certificate $ssl_fullchain;
ssl_certificate_key $ssl_key;
ssl_dhparam $dhparam;
ssl_session_cache shared:SSL:20m;
ssl_session_timeout 180m;
......@@ -49,36 +49,42 @@ server {
add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
root /home/feudal/feudalWebpage/dist;
root $dist;
index index.html;
location = /favicon.ico {
root /home/feudal/feudalWebpage/dist/assets/favicon/;
# Cache webpage assets
location /assets {
alias $dist/assets/;
expires 30d;
add_header Pragma public;
add_header Cache-Control "public";
}
# Cache static assets of the django admin
location ^~ /backend/static/ {
alias /home/feudal/feudalBackend/static/;
alias $static/;
expires 30d;
add_header Pragma public;
add_header Cache-Control "public";
}
# Shortened path for the user rest interface
location /rest {
rewrite ^/rest/(.*)$ /backend/user-api/$1;
}
# Calls to the backend are handled by uwsgi (see upstream django)
location /backend {
uwsgi_pass django;
include /home/feudal/config/uwsgi_params;
include $config/uwsgi_params;
}
# The webpage is placed at the root (including its index.html)
location /frontend {
try_files $uri $uri/ /index.html;
}
# Handle websocket
location /ws {
proxy_pass http://websocket/ws;
proxy_http_version 1.1;
......@@ -86,6 +92,7 @@ server {
proxy_set_header Connection $connection_upgrade;
}
# Redirect to direct login when we have the 'idp' parameter
location / {
if ($arg_idp) {
return 301 /backend/auth/v1/request$is_args$args;
......@@ -93,6 +100,7 @@ server {
rewrite "^$" /frontend;
}
# Cache images
location ~* \.(?:ico|css|js|gif|jpe?g|png)$ {
expires 30d;
add_header Pragma public;
......
......@@ -4,12 +4,11 @@
listeners.ssl.default = 5671
# websocket
web_stomp.port = 15674
web_stomp.port = $port_websocket
# internal stomp port
stomp.listeners.tcp.1 = 61613
### auth backends
# internal backend -> localhost authentication
......@@ -19,27 +18,20 @@ auth_backends.1 = internal
auth_backends.2 = http
# external authentication endpoints at the feudal backend
# replace domain with your domain
auth_http.http_method = post
auth_http.user_path = https://hdf-portal-dev.data.kit.edu:443/backend/auth/v1/client/user
auth_http.vhost_path = https://hdf-portal-dev.data.kit.edu:443/backend/auth/v1/client/vhost
auth_http.resource_path = https://hdf-portal-dev.data.kit.edu:443/backend/auth/v1/client/resource
auth_http.topic_path = https://hdf-portal-dev.data.kit.edu:443/backend/auth/v1/client/topic
auth_http.user_path = https://$domain:443/backend/auth/v1/client/user
auth_http.vhost_path = https://$domain:443/backend/auth/v1/client/vhost
auth_http.resource_path = https://$domain:443/backend/auth/v1/client/resource
auth_http.topic_path = https://$domain:443/backend/auth/v1/client/topic
# ssl setup (needed for amqp)
ssl_options.cacertfile = /etc/ssl/hdf-portal-dev.data.kit.edu/chain.pem
ssl_options.certfile = /etc/ssl/hdf-portal-dev.data.kit.edu/cert.pem
ssl_options.keyfile = /etc/ssl/hdf-portal-dev.data.kit.edu/key.pem
ssl_options.cacertfile = $ssl_chain
ssl_options.certfile = $ssl_cert
ssl_options.keyfile = $ssl_key
ssl_options.verify = verify_none
ssl_options.fail_if_no_peer_cert = false
# we do not use ssl for the websocket (nginx terminates tls for us)
# webstomp
# web_stomp.ssl.backlog = 1024
# web_stomp.ssl.cacertfile = /etc/ssl/hdf-portal.data.kit.edu/chain.pem
# web_stomp.ssl.certfile = /etc/ssl/hdf-portal.data.kit.edu/cert.pem
# web_stomp.ssl.keyfile = /etc/ssl/hdf-portal.data.kit.edu/key.pem
# stomp
stomp.default_user = guest
stomp.default_pass = guest
......@@ -2,11 +2,11 @@
# Django-related settings
# the base directory (full path)
chdir = /home/feudal/feudalBackend
chdir = $backend
# Django's wsgi file
module = feudal.wsgi
# the virtualenv (full path)
home = /home/feudal/feudalBackend/env
home = $venv
plugins = python35
......@@ -15,12 +15,12 @@ plugins = python35
master = true
# maximum number of worker processes
processes = 1
# the socket (use the full path to be safe
socket = /home/feudal/.feudal.sock
# the socket
socket = $uwsgi_socket
# ... with appropriate permissions - may be needed
chmod-socket = 666
# clear environment on exit
vacuum = true
logto = /home/feudal/logs/uwsgi.log
logto = $logs/uwsgi.log
......@@ -4,7 +4,7 @@ After=syslog.target
[Service]
Type=notify
ExecStart=/usr/bin/uwsgi --ini %h/config/uwsgi.ini
ExecStart=/usr/bin/uwsgi --ini $uwsgi_ini
RuntimeDirectory=uwsgi
KillSignal=SIGQUIT
StandardError=syslog
......
# vim: set filetype=bash foldmethod=marker :
# the domain / host of this machine
export domain=hdf-portal-dev.data.kit.edu
# the user which runs the backend
export user=feudal
export root=/home/$user
# dir for the config files
export config=$root/config
# the directory for the logs (unprivileged)
export logs=$root/logs
systemd=/home/$user/.local/share/systemd/user
# {{{ uwsgi
export uwsgi_ini=$config/uwsgi.ini
export uwsgi_socket=$root/feudal.sock
# }}}
# {{{ backend
# python version (currently only tested with 3.5)
export python=$(which python3.5)
# the directory into which you cloned https://git.scc.kit.edu/feudal/feudalBackend
export backend=$root/feudalBackend
export static=$backend/static
export venv=$backend/env
export secret=$config/django.key
export mysql=$config/mysql.cnf
export port_websocket=15674
# }}}
# {{{ webpage
# the directory for https://git.scc.kit.edu/feudal/feudalWebpage
export webpage=$root/feudalWebpage
export dist=$webpage/dist
# }}}
# {{{ ssl certificate
ssl=/etc/ssl/$domain
export ssl_chain=$ssl/chain.pem
export ssl_cert=$ssl/cert.pem
export ssl_fullchain=$ssl/fullchain.pem
export ssl_key=$ssl/key.pem
export dhparam=/etc/nginx/cert/dhparam.pem
# }}}
# {{{ db
export db_user=
export db_password=
export db_host=
export db_name=
# }}}
-----BEGIN DH PARAMETERS-----
MIIBCAKCAQEAgN2JcsF7n85ykYNpNKpdv7bixxjMFWkSR465VgBKTml9oPljovUc
gmLuVxATw2cepWChrxENuFkYDzpo/WDYe0VqobXJ+q17+uUea4UglfB9G7i2turP
+LdqFTzbuv7+iiJT+Q3AnquCjnpcu8fpocadSE+ZgcBHW3b2Cs8UHvxco3ywOcQh
dku2SryH445u3bg3byNQLmhZyZaypeAd44ToVZvwbs+KDbUhnzvSAYo+uN4kK00j
eZrHvbAOygqd3BJC7aWkT8GvYxlQ2kUVbMFNRfD/HJWh/2z1T3sLboes+6odSa/f
smasw3iac6Sa2B9jtmoCKsgojSEDWuFU8wIBAg==
-----END DH PARAMETERS-----
/home/feudal/.config/systemd/user/uwsgi.service
\ No newline at end of file
[client]
database =
user =
host =
password =
default-character-set = utf8
#!/bin/bash
config_dir=${config_dir-~/config}
logs_dir=${logs_dir-~/logs}
secret_key=${secret_key-./.secret.key}
# {{{ CONFIG FILES
sc=config-templates
config_file_default=./config.env.default
config_file=./config.env
echo "Using config dir: $config_dir"
echo "Using logs dir: $logs_dir"
if [[ ! -f $config_file ]]
then
echo "FEUDAL> Need file with config variables!"
echo "FEUDAL> Adjust the file $config_file_default to your needs"
echo "FEUDAL> Copy the result to $config_file"
exit 1
fi
mkdir -p $logs_dir $config_dir
cp ./example-config/home/feudal/config/* $config_dir/
mkdir -p ~/.config
cp ./example-config/home/feudal/.config/systemd $config_dir/
# load config variables
source $config_file
ln -s $config_dir/django_settings.py ./feudal/settings.py
echo
mkdir -p $logs
echo "FEUDAL> Logs will be located at: $logs"
echo "Generating secret at ./.secret.key"
[[ ! -f $secret_key ]] && \
dd if=/dev/urandom bs=64 count=1 | base64 > $secret_key
echo
mkdir -p $config
echo "FEUDAL> Config files will be located at: $config"
virtualenv -p /usr/bin/python3 ./env
source env/bin/activate
# django settings
echo "FEUDAL> Configuring $config/django_settings.py"
cat $sc/django_settings.py | envsubst > $config/django_settings.py
echo "FEUDAL> Configuring $backend/feudal/settings.py"
ln -sf $config/django_settings.py $backend/feudal/settings.py
# TODO check dependencies
pip install Django django-cors-headers djangorestframework django-mysql mysqlclient oic urllib3 pika
# django secret key
if [[ ! -f $secret ]]
then
echo "FEUDAL> Generating secret key: $secret"
dd if=/dev/urandom bs=64 count=1 | base64 > $secret
fi
# uwsgi files
echo "FEUDAL> Configuring $uwsgi_ini"
cat $sc/uwsgi.ini | envsubst > $uwsgi_ini
echo "Starting database migration (is your database configured?)"
echo "FEUDAL> Configuring $systemd/uwsgi.service"
mkdir -p $systemd
cat $sc/uwsgi.service | envsubst > $systemd/uwsgi.service
echo "FEUDAL> Configuring $config/uwsgi_params"
cp $sc/uwsgi_params $config/uwsgi_params
./manage.py makemigrations backend
./manage.py migrate
./manage.py collectstatic
echo "FEUDAL> Configuring $mysql"
cat $sc/mysql.cnf | envsubst > $mysql
echo "Creating django super user"
./manage.py createsuperuser
systemctl --user daemon-reload
echo "FEUDAL> Enabling uWSGI service (using systemctl --user enable uwsgi)"
systemctl --user enable uwsgi
# }}}
# {{{ PYTHON
echo "FEUDAL> Creating new python virtual environment: $venv"
echo "FEUDAL> Using $python"
$(which $python) -m venv $venv
echo "FEUDAL> Activating virtual environment"
source $venv/bin/activate
pip -q install -U pip
pip -q install -U Django django-cors-headers djangorestframework django-mysql django_polymorphic django-rest-polymorphic mysqlclient oic urllib3 pika django-nose ipython
# }}}
# {{{ DJANGO
echo "FEUDAL> Starting database migration"
$backend/manage.py makemigrations backend
$backend/manage.py migrate
if [[ -d $static ]]
then
rm -rf $static
mkdir -p $static
fi
echo "FEUDAL> Preparing static files in $static"
$backend/manage.py collectstatic
# }}}
# {{{ WEBPAGE
if [[ -n $webpage ]]
then
if [[ ! -d $webpage ]]
then
echo "FEUDAL> Cloning webpage into $webpage"
git clone -q https://git.scc.kit.edu/feudal/feudalWebpage $webpage
fi
echo "FEUDAL> Changing directory to $webpage"
cd $webpage
echo "FEUDAL> Installing dependencies (using npm install)"
export npm_config_loglevel=error
npm install
echo "FEUDAL> Run auditing (using npm audit fix)"
npm audit fix
ng=$webpage/node_modules/.bin/ng
echo "FEUDAL> Building webpage (using $ng build --prod)"
[[ -f $ng ]] && $ng build --prod || exit
fi
# }}}
echo "FEUDAL>"
echo "FEUDAL> Done. Please check the output above for errors"
echo "FEUDAL>"
echo "FEUDAL> Here is how you can start the backend:"
echo "FEUDAL> Run as root: systemctl start nginx rabbitmq-server"
echo "FEUDAL> Run as $user: systemctl --user start uwsgi"
echo "FEUDAL>"
echo "FEUDAL> Here is how you can check the status of the backend:"
echo "FEUDAL> Run as $user: systemctl status nginx rabbitmq-server"
echo "FEUDAL> Run as $user: systemctl --user status uwsgi"
echo "FEUDAL>"
echo "FEUDAL> Have a nice day"
# vim: set foldmethod=marker :
#!/bin/bash
if [[ $EUID -ne 0 ]]
then
echo "FEUDAL> Need root privileges"
exit 1
fi
# {{{ CONFIG FILES
sc=config-templates
config_file_default=./config.default
config_file=./config.env
if [[ ! -f $config_file ]]
then
echo "FEUDAL> Need file with config variables!"
echo "FEUDAL> Adjust the file $config_file_default to your needs"
echo "FEUDAL> Copy the result to $config_file"
exit 1
fi
# load config values
source $config_file
# write config files
nginx_dir=/etc/nginx/conf.d
mkdir -p $nginx_dir
echo "FEUDAL> Configuring $nginx_dir/feudal.conf"
# ATTENTION! nginx.conf contains variables we dont want to substitute!
# We can tell envsubst which variables to substitute
cat $sc/nginx.conf | envsubst '\$dist \$static \$uwsgi_socket \$port_websocket \$ssl_chain \$ssl_fullchain \$ssl_key \$dhparam \$domain \$config' > $nginx_dir/feudal.conf
rabbitmq_dir=/etc/rabbitmq
mkdir -p $rabbitmq_dir
echo "FEUDAL> Configuring $rabbitmq_dir/rabbitmq.conf"
cat $sc/rabbitmq.conf | envsubst > $rabbitmq_dir/rabbitmq.conf
# }}}
# {{{ APT
echo "FEUDAL> Installing dependencies"
apt install nginx rabbitmq-server uwsgi-plugin-python3 python3 python3-venv default-libmysqlclient-dev gcc nodejs
# }}}
# {{{ NGINX
echo "FEUDAL> Generating diffie hellman parameters"
openssl dhparam -out $dhparam 2048
# }}}
# {{{ RABBITMQ
echo "FEUDAL> Activating RabbitMQ plugins"
rabbitmq-plugins enable rabbitmq_web_stomp rabbitmq_stomp rabbitmq_auth_backend_http
# }}}
# {{{ SYSTEMD
echo "FEUDAL> Enabling nginx service"
echo "FEUDAL> Enabling rabbitmq-server service"
systemctl enable nginx rabbitmq-server
# }}}
echo "FEUDAL> Done."
echo "FEUDAL>"
echo "FEUDAL> Proceed by running as $user: ./install"
# vim: set foldmethod=marker :
#!/bin/bash
systemctl --no-pager status nginx rabbitmq-server
systemctl --no-pager --user status uwsgi
echo
echo
echo "FEUDAL> The status of the above services should be 'active (running)'"
#!/bin/bash
config_dir=${config_dir-~/config}
repo=${repo-~/feudalBackend}
cd $repo
rm -rf feudal/settings.py env
for file in $(ls example-config/home/feudal/config);
do
rm $config_dir/$file
done
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment