Skip to content

GitLab

Commit 7a1e1769 authored by lukas.burgey's avatar lukas.burgey
Browse files

Merge branch 'ci' into dev

parents 7ddca5dd c0ef9ca6
Pipeline #80202 passed with stage
in 1 minute and 6 seconds
Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml 0 → 100644
View file @ 7a1e1769
image: localhost:5000/feudalbackend
services:
- mysql:5.7
stages:
- test
variables:
MYSQL_DATABASE: "feudal"
MYSQL_ROOT_PASSWORD: "feudal"
test:
stage: test
script:
- cd $CI_PROJECT_DIR
- pytest --ds=feudal.docker_settings -x
Dockerfile 0 → 100644
View file @ 7a1e1769
FROM python:3.6
ENV DJANGO_SETTINGS_MODULE feudal.docker_settings
RUN apt-get update && apt-get install -y --force-yes mc nano htop python python-pip netcat gettext && rm -rf /var/lib/apt/lists/*
RUN mkdir /code
WORKDIR /code
COPY requirements.txt /code/
RUN pip install --upgrade pip
RUN pip install -r requirements.txt
CMD ["bash"]
feudal/backend/models/brokers.py → feudal/backend/brokers.py
View file @ 7a1e1769
from logging import getLogger
from json import dumps
from django.db import models
from django.core.cache import cache
import pika
from pika import BlockingConnection, ConnectionParameters, BasicProperties
from pika.credentials import PlainCredentials
from django.conf import settings
LOGGER = getLogger(__name__)
def publish_to_user(user, obj):
from . import serializers
RabbitMQInstance.load().publish_to_user(
user,
dumps(serializers.UpdateSerializer(obj).data),
)
# singleton for simple configs
# https://steelkiwi.com/blog/practical-application-singleton-design-pattern/
class SingletonModel(models.Model):
class Meta:
abstract = True
def set_cache(self):
cache.set(self.__class__.__name__, self)
# pylint: disable=invalid-name, arguments-differ
def save(self, *args, **kwargs):
self.pk = 1
super(SingletonModel, self).save(*args, **kwargs)
self.set_cache()
@classmethod
def load(cls):
if cache.get(cls.__name__) is None:
obj, created = cls.objects.get_or_create(pk=1)
if not created:
obj.set_cache()
return cache.get(cls.__name__)
# The rabbitmq instance located on this host (localhost)
class RabbitMQInstance(SingletonModel):
class RabbitMQInstance:
@property
def host(self):
......@@ -77,7 +41,7 @@ class RabbitMQInstance(SingletonModel):
def _params(self):
# we set NO port here, we use the default (probably 5672)
# this requires the
return pika.ConnectionParameters(
return ConnectionParameters(
host=self.host,
virtual_host=self.vhost,
credentials=PlainCredentials(
......@@ -95,12 +59,16 @@ class RabbitMQInstance(SingletonModel):
def _open_connection(self):
try:
# start a new connection
return pika.BlockingConnection(self._params)
return BlockingConnection(self._params)
except Exception as e:
LOGGER.exception('RabbitMQ connection error: %s', e)
raise e
def _publish(self, exchange, routing_key, body):
# when running CI tests we have no broker, so just do nothing
if getattr(settings, 'NO_BROKER', False):
return
connection = self._open_connection()
channel = connection.channel()
......@@ -111,7 +79,7 @@ class RabbitMQInstance(SingletonModel):
exchange=exchange,
routing_key=routing_key,
body=body,
properties=pika.BasicProperties(
properties=BasicProperties(
delivery_mode=1,
),
)
......@@ -123,17 +91,20 @@ class RabbitMQInstance(SingletonModel):
# called on client registration to make sure the exchanges exists
def initialize(self):
# when running CI tests we have no broker, so just do nothing
if getattr(settings, 'NO_BROKER', False):
return
connection = self._open_connection()
channel = connection.channel()
self._init_exchanges(channel)
connection.close()
def publish_deployment_state(self, deployment_state):
from .serializers.clients import DeploymentStateSerializer
def publish_deployment_state(self, deployment_state, msg):
self._publish(
'services',
deployment_state.service.name,
dumps(DeploymentStateSerializer(deployment_state).data),
msg,
)
def publish_to_user(self, user, msg):
......@@ -142,8 +113,3 @@ class RabbitMQInstance(SingletonModel):
str(user.id),
msg,
)
# we keep this, as removing this breaks migrations
def exchanges_default():
return []
feudal/backend/migrations/0001_initial.py
View file @ 7a1e1769
......@@ -8,9 +8,9 @@ import django.db.models.deletion
import django.utils.timezone
import django_mysql.models
import feudal.backend.models.auth
import feudal.backend.models.brokers
import feudal.backend.models.deployments
import feudal.backend.models.users
import feudal.backend.brokers
class Migration(migrations.Migration):
......@@ -110,7 +110,7 @@ class Migration(migrations.Migration):
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('host', models.CharField(default='localhost', max_length=150)),
('vhost', models.CharField(default='/', max_length=150)),
('exchanges', django_mysql.models.JSONField(blank=True, default=feudal.backend.models.brokers.exchanges_default, null=True)),
('exchanges', django_mysql.models.JSONField(blank=True, default=None, null=True)),
('port', models.IntegerField(default=15672)),
('username', models.CharField(default='guest', max_length=150)),
('password', models.CharField(default='guest', max_length=150)),
......
feudal/backend/models/deployments.py
View file @ 7a1e1769
from json import dumps
from logging import getLogger
from django.conf import settings
......@@ -11,9 +12,9 @@ from django_mysql.models import JSONField
from polymorphic.models import PolymorphicModel
from feudal.backend.models import Site, Service
from feudal.backend.models.brokers import publish_to_user, RabbitMQInstance
from feudal.backend.models.users import User, SSHPublicKey
from feudal.backend.models.auth.vos import VO
from feudal.backend.brokers import RabbitMQInstance
LOGGER = getLogger(__name__)
......@@ -172,11 +173,10 @@ class Deployment(PolymorphicModel):
if settings.DEBUG_PUBLISHING:
LOGGER.debug(self.msg('publish_to_user: {}'.format(self.state_target)))
publish_to_user(
from feudal.backend.models.serializers import UpdateSerializer
RabbitMQInstance().publish_to_user(
self.user,
{
'deployment': self,
},
dumps(UpdateSerializer({'deployment': self}).data),
)
def publish(self):
......@@ -592,11 +592,10 @@ class DeploymentState(models.Model):
if settings.DEBUG_PUBLISHING:
LOGGER.debug(self.msg('publish_to_user'))
publish_to_user(
from feudal.backend.models.serializers import UpdateSerializer
RabbitMQInstance().publish_to_user(
self.user,
{
'deployment_state': self,
},
dumps(UpdateSerializer({'deployment_state': self}).data),
)
def publish_to_client(self):
......@@ -618,7 +617,11 @@ class DeploymentState(models.Model):
if settings.DEBUG_PUBLISHING:
LOGGER.debug(self.msg('publish_to_client'))
RabbitMQInstance.load().publish_deployment_state(self)
from feudal.backend.models.serializers.clients import DeploymentStateSerializer
RabbitMQInstance().publish_deployment_state(
self,
dumps(DeploymentStateSerializer(self).data),
)
def msg(self, msg):
return ' {} - {}'.format(self, msg)
......
feudal/backend/models/serializers/clients.py
View file @ 7a1e1769
......@@ -2,11 +2,10 @@
# pylint: disable=abstract-method
from django_mysql.models import JSONField
from rest_framework.serializers import ModelSerializer, DictField, ListField
from rest_framework.serializers import Serializer, ModelSerializer, DictField, ListField, CharField
from rest_polymorphic.serializers import PolymorphicSerializer
from feudal.backend.models import Service
from feudal.backend.models.brokers import RabbitMQInstance
from feudal.backend.models.users import User, SSHPublicKey
from feudal.backend.models.deployments import Deployment, VODeployment, ServiceDeployment, DeploymentState, CredentialState
from feudal.backend.models.auth.serializers.clients import VOSerializer
......@@ -49,12 +48,8 @@ class UserSerializer(ModelSerializer):
]
class RabbitMQInstanceSerializer(ModelSerializer):
class Meta:
model = RabbitMQInstance
fields = [
'vhost',
]
class RabbitMQInstanceSerializer(Serializer):
vhost = CharField()
DEPLOYMENT_FIELDS = (
......
feudal/backend/models/test_deployments.py
View file @ 7a1e1769
......@@ -8,6 +8,7 @@ from feudal.backend.models.deployments import (
Deployment, get_deployment,
DEPLOYED, NOT_DEPLOYED, DEPLOYMENT_PENDING, REMOVAL_PENDING,
)
import feudal.backend.brokers
LOGGER = logging.getLogger(__name__)
......
feudal/backend/views/auth/clients.py
View file @ 7a1e1769
......@@ -9,8 +9,8 @@ from django.contrib.sessions.models import Session
from feudal.backend.models import Site, Service
from feudal.backend.models.users import User
from feudal.backend.models.brokers import RabbitMQInstance
from feudal.backend.models.auth.vos import Group, Entitlement
from feudal.backend.brokers import RabbitMQInstance
LOGGER = logging.getLogger(__name__)
......@@ -19,7 +19,7 @@ DENY = HttpResponse('deny')
def _valid_vhost(request):
if request.POST.get('vhost') == RabbitMQInstance.load().vhost:
if request.POST.get('vhost') == RabbitMQInstance().vhost:
return True
LOGGER.error('illegal vhost requested')
return False
......@@ -139,7 +139,7 @@ def _resource_authorized_apiclient(request):
and name.startswith('amq.gen-')
) or (
resource == 'exchange'
and name in RabbitMQInstance.load().exchanges
and name in RabbitMQInstance().exchanges
and 'write' not in permission
)
......
feudal/backend/views/auth/test_clients.py
View file @ 7a1e1769
from django.test import TestCase, override_settings
from django.test.client import Client
from feudal.backend.models.auth.vos import Group, Entitlement
from feudal.backend.models.brokers import RabbitMQInstance
from feudal.backend.models.users import User
from feudal.backend.models.auth import OIDCConfig
from feudal.backend.models import Site, Service
from feudal.backend.brokers import RabbitMQInstance
@override_settings(DEBUG=True, DEBUG_AUTH=True)
......@@ -45,7 +44,7 @@ class AuthorizationTestCase(TestCase):
cls.service.vos.add(cls.group)
cls.service.vos.add(cls.entitlement)
cls.broker = RabbitMQInstance.load()
cls.broker = RabbitMQInstance()
def setUp(self):
self.client = Client()
......
feudal/backend/views/clients.py
View file @ 7a1e1769
......@@ -12,8 +12,8 @@ from rest_framework.response import Response
from feudal.backend.models import Site, Service
from feudal.backend.models.auth.vos import VO, Group, Entitlement
from feudal.backend.models.deployments import NOT_DEPLOYED
from feudal.backend.models.brokers import RabbitMQInstance
from feudal.backend.models.serializers import clients
from feudal.backend.brokers import RabbitMQInstance
from feudal.backend.permissions import DownstreamOnly
LOGGER = logging.getLogger(__name__)
......@@ -182,7 +182,7 @@ class ConfigurationView(views.APIView):
self.apply_vos_to_services(request)
# initialize the broker, just in case
broker = RabbitMQInstance.load()
broker = RabbitMQInstance()
broker.initialize()
return Response({
......
feudal/docker_settings.py 0 → 100644
View file @ 7a1e1769
import os
# only set in docker_settings.py
NO_BROKER = True
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
DEBUG_AUTH = False
DEBUG_PUBLISHING = False
DEBUG_CREDENTIALS = False
ALLOWED_HOSTS = [
'feudal-dev.scc.kit.edu',
]
AUTH_USER_MODEL = 'backend.User'
# cookie settings
SESSION_COOKIE_AGE = 3600
SESSION_COOKIE_SECURE = True
SESSION_COOKIE_HTTPONLY = False
APPEND_SLASH = False
WSGI_APPLICATION = 'feudal.wsgi.application'
TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
CSRF_HEADER_NAME = 'HTTP_X_CSRFTOKEN'
# http parameter for the idp hint redirect
HTTP_PARAM_IDPHINT = 'idp'
CORS_ORIGIN_ALLOW_ALL = True
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = '9!t5e8b284qqyg4yv$-h&x1&2g&%2r%7srn=sx+3@%vyq7@1rd'
# Application definition
ROOT_URLCONF = 'feudal.backend.urls'
OIDC_REDIRECT_URI = 'https://feudal-dev.scc.kit.edu/auth/callback'
STATIC_URL = '/static/'
STATIC_ROOT = '/var/www/feudal/django-static'
INSTALLED_APPS = [
'django.contrib.admin',
'django.contrib.auth',
'polymorphic',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'rest_framework',
'feudal.backend',
'corsheaders',
'django_mysql',
'django_nose',
]
MIDDLEWARE = [
'django.middleware.security.SecurityMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'corsheaders.middleware.CorsMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
]
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [],
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [
'django.template.context_processors.debug',
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
'django.contrib.messages.context_processors.messages',
],
},
},
]
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.mysql',
'NAME': 'feudal',
'USER': 'root',
'PASSWORD': 'feudal',
'HOST': 'mysql',
}
}
# AUTHENTICATION AND AUTHORIZATION
AUTHENTICATION_BACKENDS = [
'feudal.backend.auth.OIDCTokenAuthBackend',
'django.contrib.auth.backends.ModelBackend',
]
REST_FRAMEWORK = {
'DEFAULT_AUTHENTICATION_CLASSES': [
'rest_framework.authentication.SessionAuthentication',
'feudal.backend.auth.OIDCTokenAuthHTTPBackend',
],
'DEFAULT_PERMISSION_CLASSES': [
'rest_framework.permissions.IsAuthenticated',
],
}
# Password validation
AUTH_PASSWORD_VALIDATORS = [
{
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
},
]
# Internationalization
LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'Europe/Berlin'
USE_TZ = True
USE_I18N = True
USE_L10N = True
# LOGGING
AUDIT_LOG_LEVEL = 25
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Imprint