Skip to content
GitLab
Commit d84890f9 authored by Lukas Burgey's avatar Lukas Burgey
Browse files

Implement simultaneous VO and Service Deployments 

Makes Deployment and DeploymentState many to many.
parent 5197f6c6
Hide whitespace changes
Inline Side-by-side
feudal/backend/migrations/0028_auto_20181128_1400.py 0 → 100644
View file @ d84890f9
# Generated by Django 2.1.3 on 2018-11-28 13:00
from django.conf import settings
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('backend', '0027_auto_20181126_1314'),
]
operations = [
migrations.RemoveField(
model_name='deploymentstate',
name='parent',
),
migrations.AddField(
model_name='deploymentstate',
name='deployments',
field=models.ManyToManyField(related_name='states', to='backend.Deployment'),
),
migrations.AlterField(
model_name='deploymentstate',
name='service',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='states', to='backend.Service'),
),
migrations.AlterField(
model_name='deploymentstate',
name='site',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='states', to='backend.Site'),
),
migrations.AlterField(
model_name='deploymentstate',
name='user',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='states', to=settings.AUTH_USER_MODEL),
),
]
feudal/backend/models/__init__.py
View file @ d84890f9
......@@ -43,15 +43,19 @@ class Site(models.Model):
# ignores orphaned deployment states
dep_states = [
state
for state in self.state_items.all()
for state in self.states.all()
if (
state.is_pending or state.is_credential_pending
) and state.parent is not None
) and state.deployments.exists()
]
# make the deployments unique here
for item in dep_states:
deployments[item.parent.id] = item.parent
for state in dep_states:
# we filter for deployments of the same state_target here
# if we have multiple deployments per deployment state this causes only the
# deployments with the same state_target to be pending
for deployment in state.deployments.filter(state_target=state.state_target):
deployments[deployment.id] = deployment
return deployments.values()
......
feudal/backend/models/deployments.py
View file @ d84890f9
......@@ -4,6 +4,7 @@ from logging import getLogger
from django.conf import settings
from django.db import models
from django.db.models import Q
from django_mysql.models import JSONField
from polymorphic.models import PolymorphicModel
......@@ -86,9 +87,9 @@ class Deployment(PolymorphicModel):
@property
def state(self):
if self.state_items.exists():
if self.states.exists():
_state = ''
for state in self.state_items.all():
for state in self.states.all():
if _state == '':
_state = state.state
elif _state != state.state:
......@@ -104,44 +105,69 @@ class Deployment(PolymorphicModel):
return self.state_target == self.state
def _set_target(self, target):
LOGGER.debug(self.msg('Target changed to '+target))
if str(self.state_target) == str(target):
return
self.state_target = target
LOGGER.debug(self.msg('Target: {} -> {} '.format(self.state_target, target)))
for credential_state in CredentialState.objects.filter(
target__parent=self,
):
credential_state.set_target(target)
self.state_target = target
self.save()
# FIXME this is breaking things when one wants deploy and another gets a user_remove
# # set the target to all credentials
# for state in self.states.all():
# state.assure_credential_states_exist()
# for credential_state in CredentialState.objects.filter(
# target__deployments=self,
# ):
# credential_state.set_target(target)
# Deployment.user_deploy
def user_deploy(self):
self._set_target('deployed')
for item in self.state_items.all():
LOGGER.debug(self.msg('user_deploy'))
self._set_target(DEPLOYED)
# states which are not DEPLOYED
for item in self.states.filter(~Q(state=DEPLOYED)):
item.user_deploy()
self.publish_to_client()
# each state item publishes its state to the user
self.publish()
# Deployment.user_remove
def user_remove(self):
for item in self.state_items.all():
if item.state != NOT_DEPLOYED:
item.user_remove()
LOGGER.debug(self.msg('user_remove'))
can_publish = False
self._set_target(NOT_DEPLOYED)
self.publish_to_client()
# each state item publishes its state to the user
# states which are not NOT_DEPLOYED
for item in self.states.filter(~Q(state=NOT_DEPLOYED)):
if item.user_remove():
can_publish = True
# we only publish to the clients if allowed
# we are not allowed to publish a removal if another deployment for our
# DeploymentStates exists and has the target DEPLOYED
if can_publish:
self.publish()
else:
self.publish_to_user()
def user_credential_added(self, key):
for item in self.state_items.all():
for item in self.states.all():
item.user_credential_added(key)
self.publish()
def user_credential_removed(self, key):
for item in self.state_items.all():
for item in self.states.all():
item.user_credential_removed(key)
self.publish()
def publish_to_client(self):
LOGGER.debug(self.msg('publish_to_client'))
from .serializers import clients
data = clients.DeploymentSerializer(self).data
msg = dumps(data)
......@@ -156,6 +182,8 @@ class Deployment(PolymorphicModel):
if self.user is None:
return
LOGGER.debug(self.msg('publish_to_user'))
from . import serializers
msg = dumps({
'deployment': serializers.DeploymentSerializer(self).data,
......@@ -198,13 +226,13 @@ class VODeployment(Deployment):
def routing_key(self):
return self.vo.name
def create_state_items(self):
def create_states(self):
for service in self.services:
DeploymentState.get_state_item(
self,
self.user,
service.site,
service,
deployments=[self],
)
@classmethod
......@@ -214,7 +242,7 @@ class VODeployment(Deployment):
user=user,
vo=vo,
)
deployment.create_state_items()
deployment.create_states()
return deployment
......@@ -225,7 +253,7 @@ class VODeployment(Deployment):
)
deployment.save()
deployment.create_state_items()
deployment.create_states()
LOGGER.debug(deployment.msg('Created'))
return deployment
......@@ -233,12 +261,12 @@ class VODeployment(Deployment):
def service_added(self, service):
LOGGER.debug(self.msg('Adding service {}'.format(service)))
item = DeploymentState.get_state_item(
self,
self.user,
service.site,
service,
deployments=[self],
)
if self.state_target == 'deployed':
if str(self.state_target) == 'deployed':
item.user_deploy()
def service_removed(self, service):
......@@ -272,12 +300,11 @@ class ServiceDeployment(Deployment):
return self.service.name
def create_state_item(self):
LOGGER.debug('create_state_item: creating DeploymentState for service %s at site %s', self.service, self.service.site)
DeploymentState.get_state_item(
self,
self.user,
self.service.site,
self.service,
deployments=[self],
)
@classmethod
......@@ -315,18 +342,14 @@ class ServiceDeployment(Deployment):
class DeploymentState(models.Model):
parentless = ValueError('Tried to access parent of parentless deployment state')
parent = models.ForeignKey(
deployments = models.ManyToManyField(
Deployment,
related_name='state_items',
on_delete=models.SET_NULL,
null=True
related_name='states',
)
user = models.ForeignKey(
User,
related_name='state_items',
related_name='states',
on_delete=models.SET_NULL,
blank=True,
null=True,
......@@ -334,13 +357,13 @@ class DeploymentState(models.Model):
site = models.ForeignKey(
Site,
related_name='state_items',
related_name='states',
on_delete=models.CASCADE,
)
service = models.ForeignKey(
Service,
related_name='state_items',
related_name='states',
on_delete=models.CASCADE,
)
......@@ -371,15 +394,23 @@ class DeploymentState(models.Model):
blank=True,
)
@property
def state_target(self):
for deployment in self.deployments.all():
if deployment.state_target == DEPLOYED:
return DEPLOYED
return NOT_DEPLOYED
@property
def is_pending(self):
# TODO
# pending because we are orphaned -> pending until removed everywhere
if self.parent is None:
if not self.deployments.exists():
return True
# pending because the state target is not reached
if self.parent.state_target != self.state:
if self.state_target != self.state:
return True
return False
......@@ -396,24 +427,30 @@ class DeploymentState(models.Model):
return self.user.credentials
@classmethod
def get_state_item(cls, parent, user, site, service):
def get_state_item(cls, user, site, service, deployments=[]):
try:
item = cls.objects.get(
parent=parent,
user=user,
site=site,
service=service,
)
for deployment in deployments:
if not item.deployments.filter(id=deployment.id).exists():
LOGGER.debug(item.msg('Binding to deployment {}'.format(deployment)))
item.deployments.add(deployment)
return item
except cls.DoesNotExist:
item = cls(
parent=parent,
user=user,
site=site,
service=service,
)
item.save()
for deployment in deployments:
item.deployments.add(deployment)
LOGGER.debug(item.msg('Created'))
return item
......@@ -441,14 +478,20 @@ class DeploymentState(models.Model):
# STATE TRANSITIONS
# user: provisioning requested
# DeploymentState.user_deploy
def user_deploy(self):
if self.state == REMOVAL_PENDING:
LOGGER.debug(self.msg('user_deploy'))
self._assure_credential_states_exist()
for cred_state in self.credential_states.all():
cred_state.set_target(DEPLOYED)
if str(self.state) == REMOVAL_PENDING:
self._set_state(DEPLOYED)
return
if self.state == DEPLOYED:
LOGGER.info(self.msg('ignoring invalid state transition user_deploy'))
if str(self.state) == DEPLOYED:
LOGGER.debug(self.msg('State: already deployed'))
return
self._set_state(
......@@ -456,46 +499,56 @@ class DeploymentState(models.Model):
publish=False, # the post response already contains the update
)
# user: deprovisioning requested
# DeploymentState.user_remove
# returns True if no other deployment needs this state_item to be deployed
def user_remove(self):
if self.parent is None:
LOGGER.error('user_remove: parentless')
return
if str(self.state_target) == DEPLOYED:
LOGGER.debug(self.msg('user_remove: Not removing: another deployment has target deployed'))
# False: signal the callee that a publish_to_client is *not* permitted
return False
if (
self.parent.state_target == DEPLOYED
and (self.state == FAILED or self.state == REJECTED)
LOGGER.debug(self.msg('user_remove'))
for cred_state in self.credential_states.all():
cred_state.set_target(NOT_DEPLOYED)
if str(self.state) == NOT_DEPLOYED:
LOGGER.debug(self.msg('State: already not_deployed'))
elif (
self.state_target == DEPLOYED
and (self.state == FAILED
or self.state == REJECTED)
):
self._reset()
self._set_state(NOT_DEPLOYED, publish=False)
return
if self.state == NOT_DEPLOYED:
LOGGER.info(self.msg('ignoring invalid state transition user_remove'))
return
# FIXME this will break if the client 'finishes' the deployment, after user_remove
if (
elif (
self.state == DEPLOYMENT_PENDING
or self.state == QUESTIONNAIRE
):
self._set_state(NOT_DEPLOYED)
return
self._set_state(
REMOVAL_PENDING,
publish=False, # the post response already contains the update
)
else:
# default: start the removal process
self._set_state(
REMOVAL_PENDING,
publish=False, # the post response already contains the update
)
# True: signal the callee that a publish_to_client is permitted
return True
# user: questionnaire answered
def user_answers(self, answers=None):
if self.parent is None:
LOGGER.error('user_answers: parentless')
if not self.deployments.exists():
LOGGER.error('user_remove: no deployments')
return
self.questionnaire = answers
self._set_state(DEPLOYMENT_PENDING, publish=False)
self.parent.publish_to_client()
self.publish_to_client()
def client_credential_states(self, credential_states):
# maps ssh key names to their state
......@@ -512,9 +565,16 @@ class DeploymentState(models.Model):
else:
credential_state.set(NOT_DEPLOYED)
# returns None on success, or a string describing an error
# client_response returns None on success, or a string describing an error
def client_response(self, output):
state = output.get('state', None)
if 'state' not in output:
return 'field "state" is missing in output'
state = output.get('state', '')
LOGGER.debug(self.msg('Client response: {}'.format(state)))
self._set_state(state)
credential_states = output.get('user_credential_states', None)
if credential_states is not None:
......@@ -523,27 +583,16 @@ class DeploymentState(models.Model):
self.message = output.get('message', '')
self.save()
if state is None:
return 'missing state in output'
# update values
if state == DEPLOYED:
self.credentials = output.get('credentials', {})
self.save()
# the client completed a deployment after the user wished to remove the deployment
if self.parent is not None and self.parent.state_target == NOT_DEPLOYED:
self.user_remove()
elif state == NOT_DEPLOYED:
# reset credentials and questionnaire
self._reset()
self.save()
# the client completed a removal after the user wished to deploy the deployment
if self.parent is not None and self.parent.state_target == DEPLOYED:
self.user_deploy()
elif state == QUESTIONNAIRE:
self.questionnaire = output.get('questionnaire', {})
self.save()
......@@ -552,9 +601,16 @@ class DeploymentState(models.Model):
elif state == FAILED:
pass
else:
return 'unknown state \''+state+'\''
return 'unknown state "{}"'.format(state)
# FIXME this apparently causes deployment loops
# is the target reached now?
if self.state_target != self.state:
LOGGER.debug(self.msg('Target {} still not reached. Publishing again'.format(
self.state_target,
)))
self.publish_to_client()
self._set_state(state)
return None
# resets all client sent values
......@@ -563,7 +619,7 @@ class DeploymentState(models.Model):
self.questionnaire = questionnaire_default()
self.message = ''
def _set_state(self, state, publish=True):
def _assure_credential_states_exist(self):
# assure all user credentials have a state
if self.user is not None:
for key in self.user.ssh_keys.all():
......@@ -575,26 +631,38 @@ class DeploymentState(models.Model):
except CredentialState.DoesNotExist:
LOGGER.error('CredentialState.DoesNotExist in _set_state')
if (
state == DEPLOYMENT_PENDING or
state == REMOVAL_PENDING or
state == FAILED
):
self.pending = True
def _set_state(self, state, publish=True):
self._assure_credential_states_exist()
if str(self.state) == str(state):
return
LOGGER.debug(self.msg('State: {} -> {} - Target: {}'.format(self.state, state, self.state_target)))
self.state = state
self.save()
LOGGER.debug(self.msg('State changed to '+self.state))
if publish and self.parent is not None:
self.parent.publish_to_user()
# publish to user
if publish and self.deployments.exists():
self.publish_to_user()
def publish_to_user(self):
for deployment in self.deployments.all():
deployment.publish_to_user()
def publish_to_client(self):
for deployment in self.deployments.all():
deployment.publish_to_client()
def msg(self, msg):
return '{} - {}'.format(self, msg)
def __str__(self):
if self.parent is not None:
if self.deployments.exists():
deployment_names = [str(deployment.id) for deployment in self.deployments.all()]
return 'DState: ({}:{}:{})#{}'.format(
self.parent.id,
','.join(deployment_names),
self.service,
self.site,
self.id,
......@@ -630,6 +698,7 @@ class CredentialState(models.Model):
default=False,
)
# TODO target is a stupid field name. Change it
target = models.ForeignKey(
DeploymentState,
related_name='credential_states',
......@@ -653,7 +722,7 @@ class CredentialState(models.Model):
credential=credential,
target=target,
state=NOT_DEPLOYED,
state_target=target.parent.state_target,
state_target=target.state_target,
)
new_state.save()
......@@ -663,17 +732,22 @@ class CredentialState(models.Model):
return new_state
def set_target(self, target):
if str(self.state_target) == str(target):
return
# state_target is locked, since we are marked for deletion
if self._credential_deleted:
return
LOGGER.debug(self.msg('Target: {} -> {}'.format(self.state_target, target)))
self.state_target = target
self.save()
if settings.DEBUG_CREDENTIALS:
LOGGER.debug(self.msg('Target changed to {}'.format(target)))
def set(self, state):
if str(self.state) == str(state):
return
if state == NOT_DEPLOYED and self._credential_deleted:
self._delete_state()
return
......@@ -681,10 +755,13 @@ class CredentialState(models.Model):
if state == self.state:
return
if settings.DEBUG_CREDENTIALS:
LOGGER.debug(self.msg('State: {} -> {}'.format(self.state, state)))
self.state = state