Commit 56172289 authored by Lukas Burgey's avatar Lukas Burgey
Browse files

Refactor Deployment/Task/Item logic

parent 762f7dd8
......@@ -42,7 +42,7 @@ class DeploymentsSerializer(serializers.Serializer):
class DeploymentStateSerializer(serializers.Serializer):
id = serializers.IntegerField()
action = serializers.CharField()
state_target = serializers.CharField()
user = UserSerializer()
service = ServiceSerializer()
key = backend_serializers.SSHPublicKeySerializer()
......
......@@ -13,12 +13,18 @@ LOGGER = logging.getLogger(__name__)
AUTHENTICATION_CLASSES = (BasicAuthentication, )
# The client fetches pending tasks
class DeploymentsView(generics.ListAPIView):
authentication_classes = AUTHENTICATION_CLASSES
serializer_class = DeploymentStateSerializer
def get_queryset(self):
return self.request.user.site.states
items = self.request.user.site.state_items.filter(
state='deployment_pending',
) | self.request.user.site.state_items.filter(
state='removal_pending',
)
return [item.tasks for item in items]
# the client has to fetch the configuration (like services etc.) here
......@@ -45,8 +51,11 @@ class AckView(views.APIView):
def delete(self, request, state_id=None):
# find the corresponding state for this item
for item in request.user.site.state_items.all():
if item.state.id == int(state_id):
item.success()
if item.parent.id == int(state_id):
if item.parent.state_target == 'deployed':
item.client_deployed()
else:
item.client_removed()
return Response({'ok': True})
# this is no critical
......@@ -62,29 +71,37 @@ class ResponseView(views.APIView):
status = output['status']
state_id = request.data['id']
LOGGER.debug('%s responded to state %s:\n%s', request.user, state_id, request.data)
# LOGGER.debug('%s responded to state %s:\n%s', request.user, state_id, request.data)
# find the corresponding state for this item
state_item = None
for item in request.user.site.state_items.all():
if item.state.id == int(state_id):
if item.parent.id == int(state_id):
state_item = item
if state_item is not None:
if status == 'success':
state_item.success(
credentials=request.data.output.get('credentials', None),
if status == 'deployed':
state_item.client_deployed(
credentials=output.get('credentials', None),
)
return Response({})
elif status == 'fail':
state_item.failed()
elif status == 'removed':
state_item.client_removed()
return Response({})
elif status == 'reject':
state_item.rejected(output['questionnaire'])
elif status == 'questionnaire':
state_item.client_questionnaire(
output['questionnaire'],
)
return Response({})
LOGGER.error("Unrecognized response status from client: %s", status)
return Response(
data={'error': 'unknown status'},
status=500,
)
LOGGER.info('%s executed the obsolete state#%s', request.user, state_id)
return Response(
data={'error': 'obsolete_state'},
......
......@@ -29,7 +29,7 @@ class DeploymentStateItemSerializer(serializers.ModelSerializer):
class Meta:
model = models.DeploymentStateItem
fields = [
'action',
'state_target',
'key',
'service',
'site',
......@@ -45,7 +45,7 @@ class DeploymentStateSerializer(serializers.ModelSerializer):
class Meta:
model = models.DeploymentState
fields = [
'action',
'state_target',
'key',
'service',
'id',
......@@ -56,8 +56,8 @@ class DeploymentSerializer(serializers.Serializer):
service = ServiceSerializer()
ssh_keys = backend_serializers.SSHPublicKeySerializer(many=True)
ssh_keys_to_withdraw = backend_serializers.SSHPublicKeySerializer(many=True)
deploys = DeploymentStateSerializer(many=True)
withdrawals = DeploymentStateSerializer(many=True)
# deploys = DeploymentStateSerializer(many=True)
# removals = DeploymentStateSerializer(many=True)
class Meta:
model = models.Deployment
......
......@@ -121,7 +121,7 @@ class DeploymentView(views.APIView):
if request_type == 'add':
deployment.deploy_key(request_key)
elif request_type == 'remove':
deployment.withdraw_key(request_key)
deployment.remove_key(request_key)
else:
return _api_error_response()
......
......@@ -20,6 +20,14 @@ LOGGER = logging.getLogger(__name__)
RABBITMQ_CONNECTION = None
STATE_CHOICES = (
('deployment_pending', 'Deployment Pending'),
('removal_pending', 'Removal Pending'),
('deployed', 'Deployed'),
('not_deployed', 'Not Deployed'),
('questionnaire', 'Questionnaire'),
)
# singleton for simple configs
# https://steelkiwi.com/blog/practical-application-singleton-design-pattern/
......@@ -47,6 +55,19 @@ class SingletonModel(models.Model):
def exchanges_default():
return []
# takes a list of states
# return '<state>' if all states are equal to '<state>'
# else it returns 'mixed'
def analyze_states(states):
_state = ''
for state in states:
if _state == '':
_state = state
elif _state != state:
return 'mixed'
return _state
# clients are registerred at rabbitmq, when they are assigned to a site
# (because we only then know what services they provide)
......@@ -187,6 +208,10 @@ class RabbitMQInstance(SingletonModel):
def user_info_default():
return {}
def questionnaire_default():
return {}
def credential_default():
return {}
class User(AbstractUser):
......@@ -331,7 +356,7 @@ class User(AbstractUser):
# FIXME: deleting the user brings problems:
# the deletion cascades down to DeploymentState and DeploymentStateItem
# but these need to be conserved so all clients withdrawals can be tracked
# but these need to be conserved so all clients removals can be tracked
LOGGER.info(self.msg('Deleting'))
self.delete()
......@@ -399,27 +424,25 @@ class Site(models.Model):
def __str__(self):
return self.name
# states which are still to be executed on this site
@property
def states(self):
state_items = self.state_items.filter(state='pending')\
| self.state_items.filter(state='failed')\
| self.state_items.filter(state='answered')
return [item.state
for item
in state_items]
class Service(models.Model):
name = models.CharField(max_length=150, unique=True)
description = models.TextField(max_length=300, blank=True)
name = models.CharField(
max_length=150,
unique=True,
)
description = models.TextField(
max_length=300,
blank=True,
)
site = models.ManyToManyField(
Site,
related_name='services')
related_name='services',
)
groups = models.ManyToManyField(
Group,
related_name='services',
blank=True)
blank=True,
)
@property
def routing_key(self):
......@@ -445,20 +468,23 @@ class SSHPublicKey(models.Model):
null=True,
)
# has the user triggered the deletion of this key
# has the user triggered the deletion of this key?
deleted = models.BooleanField(
default=False,
editable=False,
)
def msg(self, msg):
return '[SSHPublicKey:{}] {}'.format(self, msg)
def states(self):
states = []
for state in self.states.all():
states.append(state.states)
# does not directly delete the key if the key is deployed or withdrawn
# somewhere
# the receiver 'delete_withdrawn_ssh_key' does the actual deletion
def delete_key(self):
if (not self.states.exists() and not self.deployments.exists()):
# if this key is not deployed anywhere we delete it now
if analyze_states(self.states) == 'not_deployed':
LOGGER.info(self.msg('Direct deletion of key'))
self.delete()
return
......@@ -469,13 +495,12 @@ class SSHPublicKey(models.Model):
# delete implies withdrawing the key from all clients
for deployment in self.deployments.all():
deployment.withdraw_key(self)
deployment.remove_key(self)
# when a key is withdrawn by a client we try to finally delete it
def try_final_deletion(self):
if (self.deleted and not self.states.exists()):
LOGGER.info(self.msg(
'All clients have withdrawn this key. Final deletion'))
LOGGER.info(self.msg( 'All clients have withdrawn this key. Final deletion'))
self.delete()
return
......@@ -484,6 +509,9 @@ class SSHPublicKey(models.Model):
return "DELETED: {}".format(self.name)
return self.name
def msg(self, msg):
return '[SSHKey:{}] {}'.format(self, msg)
# Deployment describes the credential state per user (and site) as it is supposed to be
#
......@@ -517,7 +545,6 @@ class Deployment(models.Model):
default=True,
)
# get a deployment for a user/service.
# if it does not exist it is created
@classmethod
......@@ -534,23 +561,9 @@ class Deployment(models.Model):
service=service,
)
deployment.save()
LOGGER.debug(deployment.msg('created'))
return deployment
@property
def withdrawals(self):
return self.states.filter(action='withdraw')
@property
def deploys(self):
return self.states.filter(action='deploy')
def __str__(self):
return '{}:{}'.format(self.service, self.user)
def msg(self, msg):
return '[Deployment:{}] {}'.format(self, msg)
# deploy credentials which were deployed prior to deactivation
def activate(self):
if self.is_active:
......@@ -575,28 +588,10 @@ class Deployment(models.Model):
self.save()
for key in self.ssh_keys.all():
self._withdraw_key(key)
self._remove_key(key)
LOGGER.info(self.msg('deactivated'))
# only deploy the key
def _deploy_key(self, key):
state = DeploymentState.construct_deployment_state(
deployment=self,
key=key,
)
# publish the state
state.publish()
def _withdraw_key(self, key):
state = DeploymentState.construct_withdrawal_state(
deployment=self,
key=key,
)
# publish the state
state.publish()
# deploy key and track changes in the key lists
def deploy_key(self, key):
if not self.is_active:
......@@ -612,7 +607,7 @@ class Deployment(models.Model):
self._deploy_key(key)
# withdraw key and track changes in the key lists
def withdraw_key(self, key):
def remove_key(self, key):
if not self.is_active:
LOGGER.error(self.msg('cannot withdraw while deactivated'))
raise Exception('deployment deactivated')
......@@ -623,30 +618,40 @@ class Deployment(models.Model):
self.ssh_keys_to_withdraw.add(key)
self.save()
self._withdraw_key(key)
self._remove_key(key)
# only deploy the key
def _deploy_key(self, key):
state = DeploymentState.get_state(
deployment=self,
key=key,
)
state.save()
state.deploy()
def _remove_key(self, key):
state = DeploymentState.get_state(
deployment=self,
key=key,
)
state.save()
state.remove()
def __str__(self):
return '{}:{}'.format(self.service, self.user)
def invert_action(action):
if action == 'deploy':
return 'withdraw'
elif action == 'withdraw':
return 'deploy'
def msg(self, msg):
return '[Deployment:{}] {}'.format(self, msg)
# DeploymentState: knows:
# user, service, key, action
# user, service, key, state_target
class DeploymentState(models.Model):
ACTION_CHOICES = (
('deploy', 'deploy'),
('withdraw', 'withdraw'),
)
action = models.CharField(
max_length=10,
choices=ACTION_CHOICES,
)
key = models.ForeignKey(
SSHPublicKey,
related_name='states',
# deleting the key leaves us without references about its deployments
# we _HAVE_ to remove all deployments prior to deleting key
on_delete=models.CASCADE,
)
deployment = models.ForeignKey(
......@@ -654,136 +659,85 @@ class DeploymentState(models.Model):
related_name='states',
on_delete=models.CASCADE,
)
# TODO is this relation needed?
# State does relate to Deployment which relates to User
user = models.ForeignKey(
User,
related_name='deployment_states',
# TODO deleting the user leaves us without references about its deployments
# we _HAVE_ to remove all deployments prior to deleting site user
on_delete=models.CASCADE,
)
# which state do we currently want to reach?
state_target = models.CharField(
max_length=50,
choices=STATE_CHOICES,
default='deployed',
)
# the inverse action of this state is requirred
# so we invert the state and manage its state_items accordingly
def invert_state(self):
LOGGER.debug(self.msg('inverting'))
previous_action = self.action
self.action = invert_action(previous_action)
self.save()
pending_sites = [state.site for state in self.state_items.all()]
self.chancel_items()
# sites which already executed the state
# we have to send them an order to rollback the changes
for site in self.deployment.service.site.all():
if site not in pending_sites:
deploy = DeploymentStateItem(
state=self,
site=site,
user=self.deployment.user
)
deploy.save()
LOGGER.debug(deploy.msg('pending'))
@classmethod
def construct_deployment_state(cls, deployment, key):
# does a state exist for this key?
query = deployment.states.filter(key=key)
if query.exists():
if len(query) > 1:
raise Exception('Unexpected query result')
state = query.first()
if state.action == 'deploy':
raise Exception('Constructing deployment state when one already exists')
state.invert_state()
return state
@property
def states(self):
return [item.state for item in self.state_items.all()]
else:
#create new state
state = cls(
action='deploy',
deployment=deployment,
key=key,
user=deployment.user,
)
state.save()
LOGGER.debug(state.msg('pending'))
@property
def state(self):
return analyze_states(self.states)
# generate state items
for site in deployment.service.site.all():
deploy = DeploymentStateItem(
state=state,
site=site,
user=deployment.user
)
deploy.save()
LOGGER.debug(deploy.msg('pending'))
@property
def service(self):
return self.deployment.service
return state
@property
def target_reached(self):
return self.state_target == self.state
# get a deployment for a user/service.
# if it does not exist it is created
@classmethod
def construct_withdrawal_state(cls, deployment, key):
# does a state exist for this key?
query = deployment.states.filter(key=key)
def get_state(cls, deployment, key):
# check if a state does already exist
query = cls.objects.filter(
deployment=deployment,
key=key,
)
if query.exists():
if len(query) > 1:
raise Exception('Unexpected query result')
state = query.first()
if state.action == 'withdraw':
raise Exception('Constructing deployment state when one already exists')
return query.first()
state.invert_state()
return state
# create new state if not
state = cls(
deployment=deployment,
user=deployment.user,
key=key,
)
state.save()
LOGGER.debug(state.msg('created'))
else:
# create a new state
state = cls(
action='withdraw',
deployment=deployment,
key=key,
# generate state items
for site in deployment.service.site.all():
deploy = DeploymentStateItem(
parent=state,
user=deployment.user,
site=site,
)
state.save()
LOGGER.debug(state.msg('pending'))
# generate state items
for site in deployment.service.site.all():
deploy = DeploymentStateItem(
state=state,
site=site,
user=deployment.user
)
deploy.save()
LOGGER.debug(deploy.msg('pending'))
return state
def chancel_items(self):
for item in self.state_items.all():
item.chancel()
deploy.save()
def chancel(self):
self.chancel_items()
LOGGER.debug(self.msg("chanceled"))
self.delete()
return state
@property
def service(self):
return self.deployment.service
def __str__(self):
return "{}:{}#{}".format(
self.deployment.service,
self.action,
self.id,
)
def deploy(self):
self._set_target('deployed')
for item in self.state_items.all():
item.user_deploy()
self.publish_to_client()
self.publish_to_user()
def msg(self, msg):
return '[DState:{}] {}'.format(self, msg)
def remove(self):
self._set_target('not_deployed')
for item in self.state_items.all():
item.user_remove()
self.publish_to_client()
self.publish_to_user()
def publish(self):
def publish_to_client(self):
# mitigating circular dependencies here
from .clientapi.serializers import DeploymentStateSerializer
msg = json.dumps(DeploymentStateSerializer(self).data)
......@@ -792,9 +746,8 @@ class DeploymentState(models.Model):
self.service,
msg,
)
# update the state of the remote webpage
def send_state_update(self):
def publish_to_user(self):
from .frontend.views import user_state_dict
content = {
'user_state': user_state_dict(self.user),
......@@ -804,48 +757,25 @@ class DeploymentState(models.Model):
content,
)
def try_finished(self):
if not self.state_items.exists():
# finished sends its own message
self._finished()
# maintenance after all state items are done
def _finished(self):
LOGGER.info(self.msg('done'))