Commit 3a72b221 authored by lukasburgey's avatar lukasburgey
Browse files

Merge branch 'deployment-mode' into dev

parents f43065aa af65c461
Pipeline #110857 passed with stage
in 1 minute and 24 seconds
......@@ -10,7 +10,7 @@ from rest_framework.test import APIClient
from feudal.backend.brokers import RabbitMQInstance
from feudal.backend.models import Site, Service
from feudal.backend.models.deployments import get_deployment, DEPLOYED
from feudal.backend.models.deployments import DEPLOYED, VODeployment, ServiceDeployment
from feudal.backend.models.users import User
from feudal.backend.models.auth import OIDCConfig
from feudal.backend.models.auth.vos import VO, Group, Entitlement
......@@ -187,7 +187,7 @@ def service(site, group, entitlement, vo):
# a deployment with a state
@pytest.fixture
def dep(user, service):
return get_deployment(user, service=service)
return ServiceDeployment.get_or_create(user, service)
@pytest.fixture
def dep_state(dep):
......@@ -261,9 +261,9 @@ def _typed_deployed_deployment(deployment_type, user, service, vo, downstream_te
# get a deployment for the user
dep = None
if deployment_type == 'vo':
dep = get_deployment(user, vo=vo)
dep = VODeployment.get_or_create(user, vo)
else:
dep = get_deployment(user, service=service)
dep = ServiceDeployment.get_or_create(user, service)
assert dep is not None
......@@ -320,10 +320,10 @@ def _typed_pending_deployment(deployment_type, user, service, vo):
# get a deployment for the user
dep = None
if deployment_type == 'vo':
dep = get_deployment(user, vo=vo)
dep = VODeployment.get_or_create(user, vo)
assert dep.vo == vo
else:
dep = get_deployment(user, service=service)
dep = ServiceDeployment.get_or_create(user, service)
assert dep is not None
......
# Generated by Django 3.0.8 on 2020-09-24 09:38
from django.conf import settings
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('backend', '0020_auto_20200922_1333'),
]
operations = [
migrations.CreateModel(
name='UserPreferences',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('deployment_mode', models.CharField(choices=[('both', 'Pick a combination of services and VOs for deployment.'), ('services-only', 'Pick individual services for deployment'), ('vos-only', 'Pick individual VOs for deployment')], default='both', max_length=20)),
('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='preferences', to=settings.AUTH_USER_MODEL)),
],
),
]
......@@ -61,18 +61,6 @@ def user_info_default(): # pragma: no cover
return {}
def get_deployment(user, vo=None, service=None):
if vo is not None and service is None:
# get_deployment updates automatically
return VODeployment.get_deployment(user, vo)
if service is not None and vo is None:
# get_deployment updates automatically
return ServiceDeployment.get_deployment(user, service)
raise ValueError("Exactly one of 'vo' or 'service' must be set")
class Deployment(PolymorphicModel):
user = models.ForeignKey(
User,
......@@ -111,48 +99,32 @@ class Deployment(PolymorphicModel):
def target_reached(self):
return self.state_target == self.state
def _assure_states_exist(self): # pragma: no cover
raise NotImplementedError('Should be overwritten in subclass')
def update(self):
self._assure_states_exist()
self.publish_to_user()
# call when you changed Deployment.state_target
def target_changed(self):
LOGGER.debug(self.msg('Target changed to {}'.format(self.state_target)))
self._assure_states_exist()
for item in self.states.all():
item.dep_target_changed()
self.publish_to_user()
# publish if there are any pending states
for item in self.states.all():
if item.is_pending or item.is_credential_pending:
self.publish_to_client()
return # publish only once
def user_credential_added(self, key):
for item in self.states.all():
item.user_credential_added(key)
if self.state_target == DEPLOYED:
self.publish()
# TODO
# self.publish()
pass
def user_credential_removed(self, key):
for item in self.states.all():
item.user_credential_removed(key)
if self.state_target == DEPLOYED:
self.publish()
def publish_to_client(self):
for state in self.states.all():
state.publish_to_client()
# TODO
# self.publish()
pass
# sends a state update via RabbitMQ / STOMP to the users webpage instance
def publish_to_user(self):
......@@ -164,10 +136,6 @@ class Deployment(PolymorphicModel):
dumps(UpdateSerializer({'deployment': self}).data),
)
def publish(self):
self.publish_to_user()
self.publish_to_client()
def msg(self, msg): # pragma: no cover
try:
return '{} - {}'.format(self, msg)
......@@ -192,20 +160,8 @@ class VODeployment(Deployment):
def services(self):
return self.vo.services.all()
# _assure_states_exist creates missing DeploymentState for this deployment
# returns True if new states were created
def _assure_states_exist(self):
created_new_states = False
for service in self.services:
_, created = DeploymentState.get_or_create(self.user, service, self)
if created:
created_new_states = True
return created_new_states
@classmethod
def get_deployment(cls, user, vo):
def get_or_create(cls, user, vo):
deployment, created = cls.objects.get_or_create(
user=user,
vo=vo,
......@@ -213,7 +169,12 @@ class VODeployment(Deployment):
if created:
LOGGER.debug(deployment.msg('Created'))
deployment.update() # creates deployment states and publishes the creation
else:
LOGGER.debug(deployment.msg('Accessed'))
LOGGER.debug('VO has services: %s', vo.services.all())
for service in vo.services.all():
DeploymentState.get_or_create(user, service, deployment)
return deployment
......@@ -240,16 +201,16 @@ class ServiceDeployment(Deployment):
on_delete=models.CASCADE,
)
def _assure_states_exist(self):
_, created = DeploymentState.get_or_create(self.user, self.service, self)
return created
@classmethod
def get_deployment(cls, user, service):
def get_or_create(cls, user, service):
deployment, created = cls.objects.get_or_create(user=user, service=service)
if created:
LOGGER.debug(deployment.msg('Created'))
deployment.update()
else:
LOGGER.debug(deployment.msg('Accessed'))
DeploymentState.get_or_create(user, service, deployment)
return deployment
......@@ -267,7 +228,7 @@ def inform_states_about_deletion(sender, instance=None, **kwargs):
_ = kwargs
states = list(instance.states.all())
LOGGER.debug(instance.msg('Deleted deployment still has states: {}'.format(states)))
LOGGER.debug(instance.msg('Informing states about my deletion: {}'.format(states)))
instance.state_target = NOT_DEPLOYED
instance.save()
......@@ -414,7 +375,6 @@ class DeploymentState(models.Model):
return state, created
# adds the deployment to our related deployments
def bind_to_deployment(self, deployment):
if not self.deployments.filter(id=deployment.id).exists():
......@@ -458,18 +418,19 @@ class DeploymentState(models.Model):
tpart()
else:
self._set_state(tpart)
else:
LOGGER.debug(self.msg('No transition to apply for: Target = {} - State = {}'.format(self.state_target, self.state)))
# called when a client patches parts of this state
def state_changed(self):
self.audit_log_response()
LOGGER.debug(self.msg('Patched by client: {} - {}'.format(self.state, self.message)))
self.publish_to_user()
if self.state == NOT_DEPLOYED and self.is_orphaned:
# publish one last time so the webpage displays "not_deployed"
self.publish_to_user()
LOGGER.debug(self.msg('Deleting'))
# don't delete 'self' but the DepState via its id
DeploymentState.objects.get(id=self.id).delete()
return
......@@ -487,12 +448,9 @@ class DeploymentState(models.Model):
self.failed_attempts = 0
self.save()
self.publish_to_user()
# called when one of our deployments changes its state_target or when we are first created
def dep_target_changed(self):
LOGGER.debug(self.msg('Deployment changed'))
LOGGER.debug('My target: %s - My state: %s - My deps: %s', self.state_target, self.state, self.deployments.all())
self._assure_credential_states_exist()
for cred_state in self.credential_states.all():
......@@ -501,27 +459,27 @@ class DeploymentState(models.Model):
# target -> state -> transition
state_transitions = {
DEPLOYED: {
NOT_DEPLOYED: DEPLOYMENT_PENDING, # needs to get deployed
NOT_DEPLOYED: [DEPLOYMENT_PENDING, self.publish_to_client], # needs to get deployed
REMOVAL_PENDING: DEPLOYED, # aborting a removal
DEPLOYED: None, # already reached correct state
DEPLOYMENT_PENDING: None, # already on the way to the correct state
REJECTED: None, # client will not execute the deployment
FAILED: None, # the client will retry this deployment via polling
FAILED_PERMANENTLY: None, # the client will retry this deployment when it restarts
QUESTIONNAIRE: self.publish_to_user, # needs answers
QUESTIONNAIRE: None, # needs answers
},
NOT_DEPLOYED: {
DEPLOYED: [REMOVAL_PENDING, self.publish_to_user, self.publish_to_client], # needs to get removed
DEPLOYMENT_PENDING: [NOT_DEPLOYED, self.publish_to_user], # aborting a deployment
DEPLOYED: [REMOVAL_PENDING, self.publish_to_client], # needs to get removed
DEPLOYMENT_PENDING: NOT_DEPLOYED, # aborting a deployment
NOT_DEPLOYED: None, # already reached correct state
REMOVAL_PENDING: None, # already on the way to the correct state
# these four states indicate that the last deployment run was not successful
# we therefore can turn around an say that the we reached the NOT_DEPLOYED state
FAILED: [NOT_DEPLOYED, self.publish_to_user],
FAILED_PERMANENTLY: [NOT_DEPLOYED, self.publish_to_user],
REJECTED: [NOT_DEPLOYED, self.publish_to_user],
QUESTIONNAIRE: [NOT_DEPLOYED, self.publish_to_user],
FAILED: NOT_DEPLOYED,
FAILED_PERMANENTLY: NOT_DEPLOYED,
REJECTED: NOT_DEPLOYED,
QUESTIONNAIRE: NOT_DEPLOYED,
},
}
self._apply_transition(state_transitions)
......@@ -531,8 +489,8 @@ class DeploymentState(models.Model):
# state_target -> state -> transition
state_transitions = {
DEPLOYED: {
DEPLOYED: self.publish_to_client, # the user updated the answers
FAILED: self.publish_to_client, # maybe changed answers solve the problem
DEPLOYED: [DEPLOYMENT_PENDING, self.publish_to_client], # the user updated the answers
FAILED: [DEPLOYMENT_PENDING, self.publish_to_client], # maybe changed answers solve the problem
QUESTIONNAIRE: [DEPLOYMENT_PENDING, self.publish_to_client], # answers were needed for deploying
},
NOT_DEPLOYED: {
......@@ -595,6 +553,10 @@ class DeploymentState(models.Model):
self.save()
self.publish_to_user() # always publish to the use when the state changes
else:
LOGGER.debug(self.msg('State unchanged: {}'.format(self.state)))
def publish_to_user(self):
if self.user is None:
LOGGER.debug(self.msg('publish_to_user: User is None'))
......
......@@ -3,7 +3,7 @@ from rest_framework.serializers import ModelSerializer, Serializer, CharField, J
from rest_polymorphic.serializers import PolymorphicSerializer
from feudal.backend.models import Site, Service
from feudal.backend.models.users import User, SSHPublicKey
from feudal.backend.models.users import User, SSHPublicKey, UserPreferences
from feudal.backend.models.deployments import CredentialState, DeploymentState, Deployment, VODeployment, ServiceDeployment
from feudal.backend.models.auth.serializers import VOSerializer
......@@ -158,12 +158,22 @@ class DeploymentSerializer(PolymorphicSerializer):
}
class UserPreferencesSerializer(ModelSerializer):
class Meta:
model = UserPreferences
fields = (
'deployment_mode',
)
class UserStateSerializer(ModelSerializer):
deployments = DeploymentSerializer(many=True)
services = ServiceSerializer(many=True)
ssh_keys = SSHPublicKeySerializer(many=True)
states = DeploymentStateSerializer(many=True)
vos = VOSerializer(many=True)
preferences = UserPreferencesSerializer()
class Meta:
model = User
......@@ -176,6 +186,7 @@ class UserStateSerializer(ModelSerializer):
'states',
'userinfo',
'vos',
'preferences',
)
......
......@@ -5,7 +5,7 @@ import logging
from feudal.backend.tests import BaseTestCase
from feudal.backend.models import Service
from feudal.backend.models.deployments import (
Deployment, get_deployment,
Deployment, VODeployment,
DEPLOYED, NOT_DEPLOYED, DEPLOYMENT_PENDING, REMOVAL_PENDING,
)
......@@ -130,22 +130,22 @@ class DeploymentTest(BaseTestCase):
self.assertEqual(deployment.state, NOT_DEPLOYED)
def test_group_with_no_service(self):
deployment = get_deployment(self.user, self.group_none)
deployment = VODeployment.get_or_create(self.user, self.group_none)
self.deployment_run(deployment, 0)
def test_group_with_service(self):
deployment = get_deployment(self.user, vo=self.group_one)
deployment = VODeployment.get_or_create(self.user, self.group_one)
self.deployment_run(deployment, 1)
def test_group_with_two_services(self):
deployment = get_deployment(self.user, vo=self.group_two)
deployment = VODeployment.get_or_create(self.user, self.group_two)
self.deployment_run(deployment, 2)
# a vo with one service gets another service *after* the user requested the deployment
def test_group_with_delayed_service(self):
deployment = get_deployment(self.user, self.group_one)
deployment = VODeployment.get_or_create(self.user, self.group_one)
self.deployment_run_delayed_service(deployment, self.group_one, 1)
# a vo with two services loses one service *after* the user requested the deployment
def test_group_with_vanishing_service(self):
get_deployment(self.user, vo=self.group_two)
VODeployment.get_or_create(self.user, self.group_two)
......@@ -6,7 +6,7 @@ import copy
from django.urls import reverse
from feudal.backend.models import Service
from feudal.backend.models.users import User, SSHPublicKey
from feudal.backend.models.users import User, SSHPublicKey, UserPreferences
from feudal.backend.models.deployments import DEPLOYED, NOT_DEPLOYED, CredentialState, Deployment, DeploymentState
from feudal.backend.models.auth.vos import Group, Entitlement
......@@ -259,3 +259,16 @@ def test_construct_from_userinfo(idp, userinfo):
}
user = User.construct_from_userinfo(userinfo, idp)
assert user.userinfo['test_value'] == test_value
assert user.preferences.deployment_mode == UserPreferences.DEPLOYMENT_MODE_BOTH
def test_user_deletion_cascade(user):
""" the user deletion must cascade to its preferences """
prefs_pk = user.preferences.pk
assert UserPreferences.objects.filter(pk=prefs_pk).exists()
user.delete()
assert not UserPreferences.objects.filter(pk=prefs_pk).exists()
......@@ -141,6 +141,13 @@ class User(AbstractUser):
user.update_userinfo(userinfo)
user.save()
_, createdPrefs = UserPreferences.objects.get_or_create(
user=user,
)
if createdPrefs:
LOGGER.info('construct_from_userinfo: Created preferences for user %s', user)
return user
@classmethod
......@@ -449,11 +456,7 @@ class User(AbstractUser):
def delete(self, using=None, keep_parents=False):
if self.user_type == self.TYPE_CHOICE_USER:
LOGGER.info('Deleting User %s', self)
# TODO
# these deletions are a hack. django should (TM) be able to delete them itself
# but there seems to be a bug
LOGGER.info(self.msg('Preparing user deletion'))
self._delete_deployments()
......@@ -461,9 +464,7 @@ class User(AbstractUser):
for key in self.ssh_keys.all():
key.delete_key()
for state in self.states.all():
state.publish_to_client()
LOGGER.info(self.msg('Deleting right now'))
super().delete(using, keep_parents)
def add_key(self, key):
......@@ -554,3 +555,29 @@ def activate_user(sender, instance=None, created=False, **kwargs):
if instance.is_active and not instance.is_active_at_clients:
instance.activate()
class UserPreferences(models.Model):
user = models.OneToOneField(
User,
related_name='preferences',
on_delete=models.CASCADE,
)
DEPLOYMENT_MODE_BOTH = 'both'
DEPLOYMENT_MODE_SERVICES_ONLY = 'services-only'
DEPLOYMENT_MODE_VOS_ONLY = 'vos-only'
DEPLOYMENT_MODE_DEFAULT = DEPLOYMENT_MODE_BOTH
DEPLOYMENT_MODE_CHOICES = (
(DEPLOYMENT_MODE_BOTH, 'Pick a combination of services and VOs for deployment.'),
(DEPLOYMENT_MODE_SERVICES_ONLY, 'Pick individual services for deployment'),
(DEPLOYMENT_MODE_VOS_ONLY, 'Pick individual VOs for deployment'),
)
deployment_mode = models.CharField(
max_length=20,
choices=DEPLOYMENT_MODE_CHOICES,
default=DEPLOYMENT_MODE_DEFAULT,
)
......@@ -93,19 +93,3 @@ class BaseTestCase(TestCase):
user=cls.user,
).save()
# the user is logged in using session authentication
class LoggedInTest(BaseTestCase):
client = None
def setUp(self):
self.client = Client()
user = authenticate(
username=self.USER_NAME,
password=self.USER_PASSWORD,
)
self.assertIsNotNone(user)
self.client.force_login(
user=user,
)
......@@ -86,10 +86,6 @@ class ConfigurationView(views.APIView):
except VO.DoesNotExist:
service.vos.add(vo)
# we need to update the Deployments here
for dep in vo.vo_deployments.all():
dep.update()
# returns the service ID to service mapping contained in the request
def parse_sid_to_service(self, request):
self.sid_to_service = {}
......
......@@ -10,9 +10,11 @@ from rest_framework.permissions import AllowAny
from feudal.backend.views.renderers import PlainTextRenderer
from feudal.backend.models import Service
from feudal.backend.models.users import UserPreferences
from feudal.backend.models.serializers import (
UserStateSerializer, ServiceSerializer, SSHPublicKeySerializer,
DeploymentSerializer, DeploymentStateSerializer
DeploymentSerializer, DeploymentStateSerializer,
UserPreferencesSerializer
)
from feudal.backend.models.deployments import VODeployment, ServiceDeployment
from feudal.backend.models.auth.vos import VO
......@@ -83,6 +85,33 @@ class UserView(generics.RetrieveDestroyAPIView):
instance.delete()
class UserPreferencesView(generics.RetrieveUpdateAPIView):
permission_classes = PERMISSION_CLASSES
serializer_class = UserPreferencesSerializer
def get_object(self):
return self.request.user.preferences
def perform_update(self, serializer):
preferences = serializer.save()
LOGGER.debug(self.request.user.msg('Updated preferences'))
if preferences.deployment_mode == UserPreferences.DEPLOYMENT_MODE_SERVICES_ONLY:
qs = self.request.user.deployments.instance_of(VODeployment)
if qs.exists():
LOGGER.debug(self.request.user.msg('Removing my VODeployments because of a new preference'))
qs.delete()
elif preferences.deployment_mode == UserPreferences.DEPLOYMENT_MODE_VOS_ONLY:
qs = self.request.user.deployments.instance_of(ServiceDeployment)
if qs.exists():
LOGGER.debug(self.request.user.msg('Removing my ServiceDeployments because of a new preference'))
qs.delete()
class ServiceListView(generics.ListAPIView):
permission_classes = PERMISSION_CLASSES
serializer_class = ServiceSerializer
......@@ -96,8 +125,10 @@ class ServiceView(generics.RetrieveAPIView):
serializer_class = ServiceSerializer
def get_object(self):
available_services = Service.objects.filter(vos__user=self.request.user).distinct()
LOGGER.debug('Available services: %s', available_services)
return get_object_or_404(
Service.objects.filter(vos__user=self.request.user),
available_services,
id=self.kwargs['id'],
)
......@@ -180,14 +211,7 @@ class DeploymentView(generics.RetrieveUpdateAPIView):
if self.kwargs['type'] == 'vo':
try:
vo = self.request.user.vos.get(id=dep_id)
dep, created = VODeployment.objects.get_or_create(
user=self.request.user,
vo=vo,
defaults={},
)
if created:
LOGGER.debug('Created new VO deployment')
return dep
return VODeployment.get_or_create(self.request.user, vo)
except VO.DoesNotExist:
raise exceptions.ValidationError('You have no VO with id "{}"'.format(dep_id))
......@@ -197,23 +221,13 @@ class DeploymentView(generics.RetrieveUpdateAPIView):
service = None
for s in self.request.user.services:
if s.id == int(dep_id):
if service is None:
service = s
else:
raise Exception('user has multiple services with identical id')
service = s
break
if service is None: