Implement simultaneous VO and Service Deployments

Makes Deployment and DeploymentState many to many.

feudal/backend/migrations/0028_auto_20181128_1400.py

+# Generated by Django 2.1.3 on 2018-11-28 13:00
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('backend', '0027_auto_20181126_1314'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='deploymentstate',
+            name='parent',
+        ),
+        migrations.AddField(
+            model_name='deploymentstate',
+            name='deployments',
+            field=models.ManyToManyField(related_name='states', to='backend.Deployment'),
+        ),
+        migrations.AlterField(
+            model_name='deploymentstate',
+            name='service',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='states', to='backend.Service'),
+        ),
+        migrations.AlterField(
+            model_name='deploymentstate',
+            name='site',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='states', to='backend.Site'),
+        ),
+        migrations.AlterField(
+            model_name='deploymentstate',
+            name='user',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='states', to=settings.AUTH_USER_MODEL),
+        ),
+    ]

feudal/backend/models/__init__.py

@@ -43,15 +43,19 @@ class Site(models.Model):
         # ignores orphaned deployment states
         dep_states = [
             state
-            for state in self.state_items.all()
+            for state in self.states.all()
             if (
                 state.is_pending or state.is_credential_pending
-            ) and state.parent is not None
+            ) and state.deployments.exists()
         ]
 
         # make the deployments unique here
-        for item in dep_states:
-            deployments[item.parent.id] = item.parent
+        for state in dep_states:
+            # we filter for deployments of the same state_target here
+            # if we have multiple deployments per deployment state this causes only the
+            # deployments with the same state_target to be pending
+            for deployment in state.deployments.filter(state_target=state.state_target):
+                deployments[deployment.id] = deployment
 
         return deployments.values()
 

feudal/backend/models/deployments.py

@@ -4,6 +4,7 @@ from logging import getLogger
 
 from django.conf import settings
 from django.db import models
+from django.db.models import Q
 from django_mysql.models import JSONField
 from polymorphic.models import PolymorphicModel
 
@@ -86,9 +87,9 @@ class Deployment(PolymorphicModel):
 
     @property
     def state(self):
-        if self.state_items.exists():
+        if self.states.exists():
             _state = ''
-            for state in self.state_items.all():
+            for state in self.states.all():
                 if _state == '':
                     _state = state.state
                 elif _state != state.state:
@@ -104,44 +105,69 @@ class Deployment(PolymorphicModel):
         return self.state_target == self.state
 
     def _set_target(self, target):
-        LOGGER.debug(self.msg('Target changed to '+target))
+        if str(self.state_target) == str(target):
+            return
 
-        self.state_target = target
+        LOGGER.debug(self.msg('Target: {} -> {} '.format(self.state_target, target)))
 
-        for credential_state in CredentialState.objects.filter(
-                target__parent=self,
-        ):
-            credential_state.set_target(target)
+        self.state_target = target
         self.save()
 
+        # FIXME this is breaking things when one wants deploy and another gets a user_remove
+        # # set the target to all credentials
+        # for state in self.states.all():
+        #     state.assure_credential_states_exist()
+
+        # for credential_state in CredentialState.objects.filter(
+        #         target__deployments=self,
+        # ):
+        #     credential_state.set_target(target)
+
+    # Deployment.user_deploy
     def user_deploy(self):
-        self._set_target('deployed')
-        for item in self.state_items.all():
+        LOGGER.debug(self.msg('user_deploy'))
+        self._set_target(DEPLOYED)
+
+        # states which are not DEPLOYED
+        for item in self.states.filter(~Q(state=DEPLOYED)):
             item.user_deploy()
-        self.publish_to_client()
-        # each state item publishes its state to the user
 
+        self.publish()
+
+    # Deployment.user_remove
     def user_remove(self):
-        for item in self.state_items.all():
-            if item.state != NOT_DEPLOYED:
-                item.user_remove()
+        LOGGER.debug(self.msg('user_remove'))
+        can_publish = False
         self._set_target(NOT_DEPLOYED)
-        self.publish_to_client()
-        # each state item publishes its state to the user
+
+        # states which are not NOT_DEPLOYED
+        for item in self.states.filter(~Q(state=NOT_DEPLOYED)):
+            if item.user_remove():
+                can_publish = True
+
+        # we only publish to the clients if allowed
+        # we are not allowed to publish a removal if another deployment for our
+        # DeploymentStates exists and has the target DEPLOYED
+        if can_publish:
+            self.publish()
+        else:
+            self.publish_to_user()
 
     def user_credential_added(self, key):
-        for item in self.state_items.all():
+        for item in self.states.all():
             item.user_credential_added(key)
 
         self.publish()
 
     def user_credential_removed(self, key):
-        for item in self.state_items.all():
+        for item in self.states.all():
             item.user_credential_removed(key)
 
         self.publish()
 
     def publish_to_client(self):
+        LOGGER.debug(self.msg('publish_to_client'))
+
         from .serializers import clients
         data = clients.DeploymentSerializer(self).data
         msg = dumps(data)
@@ -156,6 +182,8 @@ class Deployment(PolymorphicModel):
         if self.user is None:
             return
 
+        LOGGER.debug(self.msg('publish_to_user'))
+
         from . import serializers
         msg = dumps({
             'deployment': serializers.DeploymentSerializer(self).data,
@@ -198,13 +226,13 @@ class VODeployment(Deployment):
     def routing_key(self):
         return self.vo.name
 
-    def create_state_items(self):
+    def create_states(self):
         for service in self.services:
             DeploymentState.get_state_item(
-                self,
                 self.user,
                 service.site,
                 service,
+                deployments=[self],
             )
 
     @classmethod
@@ -214,7 +242,7 @@ class VODeployment(Deployment):
                 user=user,
                 vo=vo,
             )
-            deployment.create_state_items()
+            deployment.create_states()
 
             return deployment
 
@@ -225,7 +253,7 @@ class VODeployment(Deployment):
             )
 
             deployment.save()
-            deployment.create_state_items()
+            deployment.create_states()
 
             LOGGER.debug(deployment.msg('Created'))
             return deployment
@@ -233,12 +261,12 @@ class VODeployment(Deployment):
     def service_added(self, service):
         LOGGER.debug(self.msg('Adding service {}'.format(service)))
         item = DeploymentState.get_state_item(
-            self,
             self.user,
             service.site,
             service,
+            deployments=[self],
         )
-        if self.state_target == 'deployed':
+        if str(self.state_target) == 'deployed':
             item.user_deploy()
 
     def service_removed(self, service):
@@ -272,12 +300,11 @@ class ServiceDeployment(Deployment):
         return self.service.name
 
     def create_state_item(self):
-        LOGGER.debug('create_state_item: creating DeploymentState for service %s at site %s', self.service, self.service.site)
         DeploymentState.get_state_item(
-            self,
             self.user,
             self.service.site,
             self.service,
+            deployments=[self],
         )
 
     @classmethod
@@ -315,18 +342,14 @@ class ServiceDeployment(Deployment):
 
 
 class DeploymentState(models.Model):
-    parentless = ValueError('Tried to access parent of parentless deployment state')
-
-    parent = models.ForeignKey(
+    deployments = models.ManyToManyField(
         Deployment,
-        related_name='state_items',
-        on_delete=models.SET_NULL,
-        null=True
+        related_name='states',
     )
 
     user = models.ForeignKey(
         User,
-        related_name='state_items',
+        related_name='states',
         on_delete=models.SET_NULL,
         blank=True,
         null=True,
@@ -334,13 +357,13 @@ class DeploymentState(models.Model):
 
     site = models.ForeignKey(
         Site,
-        related_name='state_items',
+        related_name='states',
         on_delete=models.CASCADE,
     )
 
     service = models.ForeignKey(
         Service,
-        related_name='state_items',
+        related_name='states',
         on_delete=models.CASCADE,
     )
 
@@ -371,15 +394,23 @@ class DeploymentState(models.Model):
         blank=True,
     )
 
+    @property
+    def state_target(self):
+        for deployment in self.deployments.all():
+            if deployment.state_target == DEPLOYED:
+                return DEPLOYED
+
+        return NOT_DEPLOYED
+
     @property
     def is_pending(self):
         # TODO
         # pending because we are orphaned -> pending until removed everywhere
-        if self.parent is None:
+        if not self.deployments.exists():
             return True
 
         # pending because the state target is not reached
-        if self.parent.state_target != self.state:
+        if self.state_target != self.state:
             return True
 
         return False
@@ -396,24 +427,30 @@ class DeploymentState(models.Model):
         return self.user.credentials
 
     @classmethod
-    def get_state_item(cls, parent, user, site, service):
+    def get_state_item(cls, user, site, service, deployments=[]):
         try:
             item = cls.objects.get(
-                parent=parent,
                 user=user,
                 site=site,
                 service=service,
             )
+
+            for deployment in deployments:
+                if not item.deployments.filter(id=deployment.id).exists():
+                    LOGGER.debug(item.msg('Binding to deployment {}'.format(deployment)))
+                    item.deployments.add(deployment)
+
             return item
 
         except cls.DoesNotExist:
             item = cls(
-                parent=parent,
                 user=user,
                 site=site,
                 service=service,
             )
             item.save()
+            for deployment in deployments:
+                item.deployments.add(deployment)
 
             LOGGER.debug(item.msg('Created'))
             return item
@@ -441,14 +478,20 @@ class DeploymentState(models.Model):
 
     # STATE TRANSITIONS
 
-    # user: provisioning requested
+    # DeploymentState.user_deploy
     def user_deploy(self):
-        if self.state == REMOVAL_PENDING:
+        LOGGER.debug(self.msg('user_deploy'))
+
+        self._assure_credential_states_exist()
+        for cred_state in self.credential_states.all():
+            cred_state.set_target(DEPLOYED)
+
+        if str(self.state) == REMOVAL_PENDING:
             self._set_state(DEPLOYED)
             return
 
-        if self.state == DEPLOYED:
-            LOGGER.info(self.msg('ignoring invalid state transition user_deploy'))
+        if str(self.state) == DEPLOYED:
+            LOGGER.debug(self.msg('State: already deployed'))
             return
 
         self._set_state(
@@ -456,46 +499,56 @@ class DeploymentState(models.Model):
             publish=False,  # the post response already contains the update
         )
 
-    # user: deprovisioning requested
+    # DeploymentState.user_remove
+    # returns True if no other deployment needs this state_item to be deployed
     def user_remove(self):
-        if self.parent is None:
-            LOGGER.error('user_remove: parentless')
-            return
+        if str(self.state_target) == DEPLOYED:
+            LOGGER.debug(self.msg('user_remove: Not removing: another deployment has target deployed'))
+
+            # False: signal the callee that a publish_to_client is *not* permitted
+            return False
 
-        if (
-                self.parent.state_target == DEPLOYED
-                and (self.state == FAILED or self.state == REJECTED)
+        LOGGER.debug(self.msg('user_remove'))
+
+        for cred_state in self.credential_states.all():
+            cred_state.set_target(NOT_DEPLOYED)
+
+        if str(self.state) == NOT_DEPLOYED:
+            LOGGER.debug(self.msg('State: already not_deployed'))
+
+        elif (
+                self.state_target == DEPLOYED
+                and (self.state == FAILED
+                     or self.state == REJECTED)
         ):
             self._reset()
             self._set_state(NOT_DEPLOYED, publish=False)
-            return
 
-        if self.state == NOT_DEPLOYED:
-            LOGGER.info(self.msg('ignoring invalid state transition user_remove'))
-            return
-
-        # FIXME this will break if the client 'finishes' the deployment, after user_remove
-        if (
+        elif (
                 self.state == DEPLOYMENT_PENDING
                 or self.state == QUESTIONNAIRE
         ):
             self._set_state(NOT_DEPLOYED)
-            return
 
-        self._set_state(
-            REMOVAL_PENDING,
-            publish=False,  # the post response already contains the update
-        )
+        else:
+            # default: start the removal process
+            self._set_state(
+                REMOVAL_PENDING,
+                publish=False,  # the post response already contains the update
+            )
+
+        # True: signal the callee that a publish_to_client is permitted
+        return True
 
     # user: questionnaire answered
     def user_answers(self, answers=None):
-        if self.parent is None:
-            LOGGER.error('user_answers: parentless')
+        if not self.deployments.exists():
+            LOGGER.error('user_remove: no deployments')
             return
 
         self.questionnaire = answers
         self._set_state(DEPLOYMENT_PENDING, publish=False)
-        self.parent.publish_to_client()
+        self.publish_to_client()
 
     def client_credential_states(self, credential_states):
         # maps ssh key names to their state
@@ -512,9 +565,16 @@ class DeploymentState(models.Model):
             else:
                 credential_state.set(NOT_DEPLOYED)
 
-    # returns None on success, or a string describing an error
+    # client_response returns None on success, or a string describing an error
     def client_response(self, output):
-        state = output.get('state', None)
+
+        if 'state' not in output:
+            return 'field "state" is missing in output'
+
+        state = output.get('state', '')
+        LOGGER.debug(self.msg('Client response: {}'.format(state)))
+
+        self._set_state(state)
 
         credential_states = output.get('user_credential_states', None)
         if credential_states is not None:
@@ -523,27 +583,16 @@ class DeploymentState(models.Model):
         self.message = output.get('message', '')
         self.save()
 
-        if state is None:
-            return 'missing state in output'
-
         # update values
         if state == DEPLOYED:
             self.credentials = output.get('credentials', {})
             self.save()
 
-            # the client completed a deployment after the user wished to remove the deployment
-            if self.parent is not None and self.parent.state_target == NOT_DEPLOYED:
-                self.user_remove()
-
         elif state == NOT_DEPLOYED:
             # reset credentials and questionnaire