Commit 2d6aa652 authored by lukasburgey's avatar lukasburgey
Browse files

Merge branch 'dev'

parents b0f2d9b3 96759d92
Pipeline #115109 passed with stage
in 1 minute and 24 seconds
......@@ -10,7 +10,7 @@ from rest_framework.test import APIClient
from feudal.backend.brokers import RabbitMQInstance
from feudal.backend.models import Site, Service
from feudal.backend.models.deployments import get_deployment, DEPLOYED
from feudal.backend.models.deployments import DEPLOYED, VODeployment, ServiceDeployment
from feudal.backend.models.users import User
from feudal.backend.models.auth import OIDCConfig
from feudal.backend.models.auth.vos import VO, Group, Entitlement
......@@ -103,6 +103,36 @@ def userinfo(idp, entitlement, group):
idp.userinfo_field_entitlements: [entitlement.name],
}
@pytest.fixture
def userinfo_without_lists(idp, entitlement, group):
return {
'iss': idp.issuer_uri,
'sub':'fb0fa558-cfa2-49f9-b847-5c651d1f6135',
'ssh_key': 'ssh-rsa AAAAB3NzaC1yhApzBpUulukg9Q== TEST_KEY',
'name': 'Gustav Holst',
'given_name': 'Gustav',
'family_name': 'Holst',
'preferred_username': 'gustav.holst',
'email': 'gustav@holst-feudal-test.co.uk',
idp.userinfo_field_groups: group.name,
idp.userinfo_field_entitlements: entitlement.name,
}
@pytest.fixture
def userinfo_invalid_vos(idp):
return {
'iss': idp.issuer_uri,
'sub':'fb0fa558-cfa2-49f9-b847-5c651d1f6135',
'ssh_key': 'ssh-rsa AAAAB3NzaC1yhApzBpUulukg9Q== TEST_KEY',
'name': 'Gustav Holst',
'given_name': 'Gustav',
'family_name': 'Holst',
'preferred_username': 'gustav.holst',
'email': 'gustav@holst-feudal-test.co.uk',
idp.userinfo_field_groups: 0, # invalid value
idp.userinfo_field_entitlements: 0, # invalid value
}
@pytest.fixture
def password():
return 'foobarbaz'
......@@ -157,7 +187,7 @@ def service(site, group, entitlement, vo):
# a deployment with a state
@pytest.fixture
def dep(user, service):
return get_deployment(user, service=service)
return ServiceDeployment.get_or_create(user, service)
@pytest.fixture
def dep_state(dep):
......@@ -231,9 +261,9 @@ def _typed_deployed_deployment(deployment_type, user, service, vo, downstream_te
# get a deployment for the user
dep = None
if deployment_type == 'vo':
dep = get_deployment(user, vo=vo)
dep = VODeployment.get_or_create(user, vo)
else:
dep = get_deployment(user, service=service)
dep = ServiceDeployment.get_or_create(user, service)
assert dep is not None
......@@ -290,10 +320,10 @@ def _typed_pending_deployment(deployment_type, user, service, vo):
# get a deployment for the user
dep = None
if deployment_type == 'vo':
dep = get_deployment(user, vo=vo)
dep = VODeployment.get_or_create(user, vo)
assert dep.vo == vo
else:
dep = get_deployment(user, service=service)
dep = ServiceDeployment.get_or_create(user, service)
assert dep is not None
......
# Generated by Django 3.0.8 on 2020-09-24 09:38
from django.conf import settings
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('backend', '0020_auto_20200922_1333'),
]
operations = [
migrations.CreateModel(
name='UserPreferences',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('deployment_mode', models.CharField(choices=[('both', 'Pick a combination of services and VOs for deployment.'), ('services-only', 'Pick individual services for deployment'), ('vos-only', 'Pick individual VOs for deployment')], default='both', max_length=20)),
('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='preferences', to=settings.AUTH_USER_MODEL)),
],
),
]
......@@ -60,11 +60,6 @@ class VO(PolymorphicModel):
def pretty_name(self):
return self.name
@property
def broker_exchange(self): # pragma: no cover
# overridden in the implementations
return None
def __str__(self):
return self.name
......@@ -100,10 +95,6 @@ class Group(VO):
def pretty_name(self):
return self.name
@property
def broker_exchange(self): # pragma: no cover
return 'groups'
def __str__(self):
return 'VO-GROUP-'+self.name
......@@ -158,10 +149,6 @@ class Entitlement(VO):
def full_name(self):
return self.__repr__()
@property
def broker_exchange(self): # pragma: no cover
return 'entitlements'
@staticmethod
def extract_name(raw_name):
name_search = re.search('^(.*)#', raw_name)
......
......@@ -7,6 +7,8 @@ from logging import getLogger
from django.conf import settings
from django.db import models
from django.db.models.signals import pre_delete
from django.dispatch import receiver
from django_mysql.models import JSONField
from polymorphic.models import PolymorphicModel
......@@ -59,24 +61,6 @@ def user_info_default(): # pragma: no cover
return {}
def get_deployment(user, vo=None, service=None):
if vo is not None and service is not None:
raise ValueError('Cannot create deployment for both vo and service')
if vo is not None:
# get_deployment updates automatically
return VODeployment.get_deployment(user, vo)
if service is not None:
# get_deployment updates automatically
return ServiceDeployment.get_deployment(user, service)
deps = Deployment.objects.filter(user=user)
for dep in deps:
dep.update()
return deps
class Deployment(PolymorphicModel):
user = models.ForeignKey(
User,
......@@ -96,11 +80,6 @@ class Deployment(PolymorphicModel):
default=True,
)
# credentials provided by the backend to the clients
@property
def credentials(self):
return self.user.credentials
@property
def state(self):
if self.states.exists():
......@@ -120,48 +99,32 @@ class Deployment(PolymorphicModel):
def target_reached(self):
return self.state_target == self.state
def _assure_states_exist(self):
raise NotImplementedError('Should be overwritten in subclass')
def update(self):
self._assure_states_exist()
self.publish_to_user()
# call when you changed Deployment.state_target
def target_changed(self):
LOGGER.debug(self.msg('Target changed to {}'.format(self.state_target)))
self._assure_states_exist()
for item in self.states.all():
item.dep_target_changed()
self.publish_to_user()
# publish if there are any pending states
for item in self.states.all():
if item.is_pending or item.is_credential_pending:
self.publish_to_client()
return # publish only once
def user_credential_added(self, key):
for item in self.states.all():
item.user_credential_added(key)
if self.state_target == DEPLOYED:
self.publish()
# TODO
# self.publish()
pass
def user_credential_removed(self, key):
for item in self.states.all():
item.user_credential_removed(key)
if self.state_target == DEPLOYED:
self.publish()
def publish_to_client(self):
for state in self.states.all():
state.publish_to_client()
# TODO
# self.publish()
pass
# sends a state update via RabbitMQ / STOMP to the users webpage instance
def publish_to_user(self):
......@@ -173,17 +136,13 @@ class Deployment(PolymorphicModel):
dumps(UpdateSerializer({'deployment': self}).data),
)
def publish(self):
self.publish_to_user()
self.publish_to_client()
def msg(self, msg):
def msg(self, msg): # pragma: no cover
try:
return '{} - {}'.format(self, msg)
except: # this call breaks sometimes when deleting from the django admin
return 'broken-deployment'
def __str__(self):
def __str__(self): # pragma: no cover
return 'Abstr. Deployment:({})#{}'.format(
self.user,
self.id,
......@@ -201,39 +160,8 @@ class VODeployment(Deployment):
def services(self):
return self.vo.services.all()
@property
def broker_exchange(self):
return self.vo.broker_exchange
@property
def routing_key(self):
return self.vo.name
@classmethod
def create(cls, *args, **kwargs):
user = kwargs.get('user')
vo = kwargs.get('vo')
if vo not in user.vos.all():
raise ValueError('User is not permitted to have a deployment of this vo!')
return cls(*args, **kwargs)
# _assure_states_exist creates missing DeploymentState for this deployment
# returns True if new states were created
def _assure_states_exist(self):
created_new_states = False
for service in self.services:
_, created = DeploymentState.get_or_create(self.user, service, self)
if created:
created_new_states = True
return created_new_states
@classmethod
def get_deployment(cls, user, vo):
def get_or_create(cls, user, vo):
deployment, created = cls.objects.get_or_create(
user=user,
vo=vo,
......@@ -241,7 +169,12 @@ class VODeployment(Deployment):
if created:
LOGGER.debug(deployment.msg('Created'))
deployment.update() # creates deployment states and publishes the creation
else:
LOGGER.debug(deployment.msg('Accessed'))
LOGGER.debug('VO has services: %s', vo.services.all())
for service in vo.services.all():
DeploymentState.get_or_create(user, service, deployment)
return deployment
......@@ -268,37 +201,16 @@ class ServiceDeployment(Deployment):
on_delete=models.CASCADE,
)
@property
def broker_exchange(self):
return 'services'
@property
def routing_key(self):
return self.service.name
@classmethod
def create(cls, *args, **kwargs):
user = kwargs.get('user')
service = kwargs.get('service')
# check if one of the users vos can be used for the service
for vo in user.vos.all():
if vo in service.vos.all():
return cls(*args, **kwargs)
raise ValueError('User is not permitted to have a deployment of this service!')
def _assure_states_exist(self):
_, created = DeploymentState.get_or_create(self.user, self.service, self)
return created
@classmethod
def get_deployment(cls, user, service):
def get_or_create(cls, user, service):
deployment, created = cls.objects.get_or_create(user=user, service=service)
if created:
LOGGER.debug(deployment.msg('Created'))
deployment.update()
else:
LOGGER.debug(deployment.msg('Accessed'))
DeploymentState.get_or_create(user, service, deployment)
return deployment
......@@ -309,6 +221,19 @@ class ServiceDeployment(Deployment):
self.id,
)
@receiver(pre_delete, sender=VODeployment)
@receiver(pre_delete, sender=ServiceDeployment)
def inform_states_about_deletion(sender, instance=None, **kwargs):
_ = sender
_ = kwargs
states = list(instance.states.all())
LOGGER.debug(instance.msg('Informing states about my deletion: {}'.format(states)))
instance.state_target = NOT_DEPLOYED
instance.save()
instance.target_changed()
class DeploymentState(models.Model):
deployments = models.ManyToManyField(
......@@ -433,31 +358,22 @@ class DeploymentState(models.Model):
@classmethod
def get_or_create(cls, user, service, deployment):
try:
state, created = cls.objects.get_or_create(
user=user,
site=service.site,
service=service,
defaults={
'userinfo': user.userinfo,
'service_name': service.name,
}
)
if created:
LOGGER.debug(state.msg('Created'))
state.bind_to_deployment(deployment)
return state, created
except cls.MultipleObjectsReturned:
LOGGER.error(
'Multiple DeploymentStates for %s and %s and %s',
user,
service,
deployment,
)
return None, False
state, created = cls.objects.get_or_create(
user=user,
service=service,
defaults={
'userinfo': user.userinfo,
'service_name': service.name,
'site': service.site,
}
)
if created:
LOGGER.debug(state.msg('Created'))
# always bind this state to the deployment
state.bind_to_deployment(deployment)
return state, created
# adds the deployment to our related deployments
def bind_to_deployment(self, deployment):
......@@ -502,18 +418,19 @@ class DeploymentState(models.Model):
tpart()
else:
self._set_state(tpart)
else:
LOGGER.debug(self.msg('No transition to apply for: Target = {} - State = {}'.format(self.state_target, self.state)))
# called when a client patches parts of this state
def state_changed(self):
self.audit_log_response()
LOGGER.debug(self.msg('Patched by client: {} - {}'.format(self.state, self.message)))
self.publish_to_user()
if self.state == NOT_DEPLOYED and self.is_orphaned:
# publish one last time so the webpage displays "not_deployed"
self.publish_to_user()
LOGGER.debug(self.msg('Deleting'))
# don't delete 'self' but the DepState via its id
DeploymentState.objects.get(id=self.id).delete()
return
......@@ -531,8 +448,6 @@ class DeploymentState(models.Model):
self.failed_attempts = 0
self.save()
self.publish_to_user()
# called when one of our deployments changes its state_target or when we are first created
def dep_target_changed(self):
LOGGER.debug(self.msg('Deployment changed'))
......@@ -544,27 +459,27 @@ class DeploymentState(models.Model):
# target -> state -> transition
state_transitions = {
DEPLOYED: {
NOT_DEPLOYED: DEPLOYMENT_PENDING, # needs to get deployed
NOT_DEPLOYED: [DEPLOYMENT_PENDING, self.publish_to_client], # needs to get deployed
REMOVAL_PENDING: DEPLOYED, # aborting a removal
DEPLOYED: None, # already reached correct state
DEPLOYMENT_PENDING: None, # already on the way to the correct state
REJECTED: None, # client will not execute the deployment
FAILED: None, # the client will retry this deployment via polling
FAILED_PERMANENTLY: None, # the client will retry this deployment when it restarts
QUESTIONNAIRE: self.publish_to_user, # needs answers
QUESTIONNAIRE: None, # needs answers
},
NOT_DEPLOYED: {
DEPLOYED: [REMOVAL_PENDING, self.publish_to_user, self.publish_to_client], # needs to get removed
DEPLOYMENT_PENDING: [NOT_DEPLOYED, self.publish_to_user], # aborting a deployment
DEPLOYED: [REMOVAL_PENDING, self.publish_to_client], # needs to get removed
DEPLOYMENT_PENDING: NOT_DEPLOYED, # aborting a deployment
NOT_DEPLOYED: None, # already reached correct state
REMOVAL_PENDING: None, # already on the way to the correct state
# these four states indicate that the last deployment run was not successful
# we therefore can turn around an say that the we reached the NOT_DEPLOYED state
FAILED: [NOT_DEPLOYED, self.publish_to_user],
FAILED_PERMANENTLY: [NOT_DEPLOYED, self.publish_to_user],
REJECTED: [NOT_DEPLOYED, self.publish_to_user],
QUESTIONNAIRE: [NOT_DEPLOYED, self.publish_to_user],
FAILED: NOT_DEPLOYED,
FAILED_PERMANENTLY: NOT_DEPLOYED,
REJECTED: NOT_DEPLOYED,
QUESTIONNAIRE: NOT_DEPLOYED,
},
}
self._apply_transition(state_transitions)
......@@ -574,8 +489,8 @@ class DeploymentState(models.Model):
# state_target -> state -> transition
state_transitions = {
DEPLOYED: {
DEPLOYED: self.publish_to_client, # the user updated the answers
FAILED: self.publish_to_client, # maybe changed answers solve the problem
DEPLOYED: [DEPLOYMENT_PENDING, self.publish_to_client], # the user updated the answers
FAILED: [DEPLOYMENT_PENDING, self.publish_to_client], # maybe changed answers solve the problem
QUESTIONNAIRE: [DEPLOYMENT_PENDING, self.publish_to_client], # answers were needed for deploying
},
NOT_DEPLOYED: {
......@@ -624,7 +539,7 @@ class DeploymentState(models.Model):
for key in self.user.ssh_keys.all():
CredentialState.get_credential_state(key, self)
def _set_state(self, state, publish_to_user=False, publish_to_client=False):
def _set_state(self, state):
self._assure_credential_states_exist()
# same state
......@@ -638,11 +553,9 @@ class DeploymentState(models.Model):
self.save()
if publish_to_user:
self.publish_to_user()
if publish_to_client:
self.publish_to_client()
self.publish_to_user() # always publish to the use when the state changes
else:
LOGGER.debug(self.msg('State unchanged: {}'.format(self.state)))
def publish_to_user(self):
if self.user is None:
......@@ -658,10 +571,6 @@ class DeploymentState(models.Model):
)
def publish_to_client(self):
# no need to publish if not pending
if not self.is_pending:
return
self.audit_log_request()
LOGGER.log(settings.DEBUGX_LOG_LEVEL, self.msg('publish_to_client'))
......
......@@ -3,7 +3,7 @@ from rest_framework.serializers import ModelSerializer, Serializer, CharField, J
from rest_polymorphic.serializers import PolymorphicSerializer
from feudal.backend.models import Site, Service
from feudal.backend.models.users import User, SSHPublicKey
from feudal.backend.models.users import User, SSHPublicKey, UserPreferences
from feudal.backend.models.deployments import CredentialState, DeploymentState, Deployment, VODeployment, ServiceDeployment
from feudal.backend.models.auth.serializers import VOSerializer
......@@ -158,12 +158,22 @@ class DeploymentSerializer(PolymorphicSerializer):
}
class UserPreferencesSerializer(ModelSerializer):
class Meta:
model = UserPreferences
fields = (
'deployment_mode',
)
class UserStateSerializer(ModelSerializer):
deployments = DeploymentSerializer(many=True)
services = ServiceSerializer(many=True)
ssh_keys = SSHPublicKeySerializer(many=True)
states = DeploymentStateSerializer(many=True)
vos = VOSerializer(many=True)
preferences = UserPreferencesSerializer()
class Meta:
model = User
......@@ -176,6 +186,7 @@ class UserStateSerializer(ModelSerializer):
'states',
'userinfo',
'vos',
'preferences',
)
......
......@@ -299,6 +299,7 @@ def test_upstream_changed_userinfo_redeploy(user, deployed_vo_deployment, upstre
assert state.userinfo['name'] == 'new name'
# }}}
# SERIALIZERS {{{
def test_deployment_state_serializer(pending_vo_deployment):
# setup a deployed deployment for the service
......@@ -307,3 +308,76 @@ def test_deployment_state_serializer(pending_vo_deployment):
ser = webpage_serializers.DeploymentStateSerializer(state)
assert ser.data is not None
# }}}
# VO-Deployment and Service-Deployment interaction {{{
def test_both_service_and_vo_deployments_remove(pending_vo_deployment, pending_service_deployment, service):
"""
The user first requests a VO-Dep which deploys a Service.
Then an additional Service-Deployment for the same Service (which actually should change nothing)
Then the VO-Deployment is removed.
The Service must still be deployed because of the Service-Deployment
"""
vo_dep, state_from_vo = pending_vo_deployment(service=service)
assert state_from_vo.is_pending
service_dep, state_from_service = pending_service_deployment(service=service)
assert state_from_service.is_pending
# these two deployments should've mapped to the same state
assert state_from_vo.pk == state_from_service.pk
state = state_from_vo # might as well just call it state from here on
# remove first dep
vo_dep.state_target = NOT_DEPLOYED
vo_dep.save()
vo_dep.target_changed()