Merge branch 'refactoring' into dev

feudal/backend/admin/__init__.py

 
-from django.contrib import admin
+from django.contrib.admin import site
 from django.contrib.auth.models import Group as AuthGroup
 
-from .. import models
-from ..auth.v1.models import OIDCConfig
-from ..auth.v1.models.vo import VO, Group, Entitlement, EntitlementNameSpace
+from feudal.backend.models import Site, Service
+from feudal.backend.models.users import User, SSHPublicKey
+from feudal.backend.models.deployments import VODeployment, ServiceDeployment, Deployment, DeploymentState, CredentialState
+from feudal.backend.models.auth import OIDCConfig
+from feudal.backend.models.auth.vos import VO, Group, Entitlement, EntitlementNameSpace
 
-from ..models import User
-from ..models.deployments import VODeployment, ServiceDeployment, Deployment, DeploymentState, CredentialState
-
-from .users import CustomUserAdmin
-from .vos import GroupAdmin, EntitlementAdmin, VOAdmin
-from .deployments import VODeploymentAdmin, ServiceDeploymentAdmin, DeploymentAdmin
+from feudal.backend.admin.users import CustomUserAdmin
+from feudal.backend.admin.vos import GroupAdmin, EntitlementAdmin, VOAdmin
+from feudal.backend.admin.deployments import VODeploymentAdmin, ServiceDeploymentAdmin, DeploymentAdmin
 
 
 # users
-admin.site.register(User, CustomUserAdmin)
-admin.site.unregister(AuthGroup)
+site.register(User, CustomUserAdmin)
+site.unregister(AuthGroup)
 
 # vos
-admin.site.register(Group, GroupAdmin)
-admin.site.register(Entitlement, EntitlementAdmin)
-admin.site.register(VO, VOAdmin)
-admin.site.register(EntitlementNameSpace)
-admin.site.register(OIDCConfig)
+site.register(Group, GroupAdmin)
+site.register(Entitlement, EntitlementAdmin)
+site.register(VO, VOAdmin)
+site.register(EntitlementNameSpace)
+site.register(OIDCConfig)
 
 # deployments
-admin.site.register(VODeployment, VODeploymentAdmin)
-admin.site.register(ServiceDeployment, ServiceDeploymentAdmin)
-admin.site.register(Deployment, DeploymentAdmin)
+site.register(VODeployment, VODeploymentAdmin)
+site.register(ServiceDeployment, ServiceDeploymentAdmin)
+site.register(Deployment, DeploymentAdmin)
 
 
 # other
-admin.site.register([
+site.register([
     DeploymentState,
     CredentialState,
-    models.SSHPublicKey,
-    models.Site,
-    models.Service,
+    SSHPublicKey,
+    Site,
+    Service,
 ])

feudal/backend/admin/deployments.py

 
 from polymorphic.admin import PolymorphicParentModelAdmin, PolymorphicChildModelAdmin
 
-from ..auth.v1.models.vo import Entitlement
-
-from ..models import deployments
+from feudal.backend.models.deployments import Deployment, VODeployment, ServiceDeployment
 
 
 class VODeploymentAdmin(PolymorphicChildModelAdmin):
@@ -15,6 +13,6 @@ class ServiceDeploymentAdmin(PolymorphicChildModelAdmin):
 
 
 class DeploymentAdmin(PolymorphicParentModelAdmin):
-    base_model = Entitlement  # Explicitly set here!
+    base_model = Deployment  # Explicitly set here!
     show_in_index = True  # makes child model admin visible in main admin site
-    child_models = (deployments.VODeployment, deployments.ServiceDeployment)
+    child_models = (VODeployment, ServiceDeployment)

feudal/backend/admin/users.py

@@ -3,9 +3,8 @@ from django.contrib.admin import SimpleListFilter
 from django.contrib.auth.admin import UserAdmin
 from django.contrib.auth.forms import UserCreationForm
 
-from ..auth.v1.models import OIDCConfig
-
-from ..models import User
+from feudal.backend.models import User
+from feudal.backend.models.auth import OIDCConfig
 
 
 class TypeFilter(SimpleListFilter):

feudal/backend/admin/vos.py

 
 from polymorphic.admin import PolymorphicParentModelAdmin, PolymorphicChildModelAdmin
 
-from ..auth.v1.models.vo import VO, Group, Entitlement
+from feudal.backend.models.auth.vos import VO, Group, Entitlement
 
 
 class GroupAdmin(PolymorphicChildModelAdmin):

feudal/backend/auth/v1/__init__.py → feudal/backend/auth.py

 
-# pylint: disable=import-outside-toplevel
-
 import logging
 from urllib.error import HTTPError
 import jwt
 
 from rest_framework.authentication import BaseAuthentication
 
-from .utils import get_session, set_session, del_session, SessionError
-from .models import OIDCConfig
+from feudal.backend.sessions import get_session, set_session, del_session, SessionError
+from feudal.backend.models.auth import OIDCConfig
+from feudal.backend.models.users import User
 
 
 LOGGER = logging.getLogger(__name__)
@@ -18,7 +17,6 @@ class OIDCTokenAuthBackend:
 
     # get_user is part of the authentication backend API
     def get_user(self, user_id):
-        from feudal.backend.models.users import User
         try:
             return User.objects.get(
                 user_type=User.TYPE_CHOICE_USER,
@@ -101,9 +99,6 @@ class OIDCTokenAuthBackend:
 
         access_token = request.META['HTTP_AUTHORIZATION']
 
-        # this import is used in the except blocks below! Don't move it!
-        from feudal.backend.models.users import User
-
         try:
             idp, userinfo = self._get_idp_userinfo(request, access_token)
 

feudal/backend/migrations/0001_initial.py

@@ -7,7 +7,7 @@ from django.db import migrations, models
 import django.db.models.deletion
 import django.utils.timezone
 import django_mysql.models
-import feudal.backend.auth.v1.models
+import feudal.backend.models.auth
 import feudal.backend.models.brokers
 import feudal.backend.models.deployments
 import feudal.backend.models.users
@@ -99,7 +99,7 @@ class Migration(migrations.Migration):
                 ('enabled', models.BooleanField(default=False)),
                 ('name', models.CharField(max_length=200)),
                 ('redirect_uri', models.CharField(default='https://<domain>/backend/auth/v1/callback', max_length=200)),
-                ('scopes', django_mysql.models.JSONField(default=feudal.backend.auth.v1.models.scopes_default, help_text='The scopes we use when requesting user infos')),
+                ('scopes', django_mysql.models.JSONField(default=feudal.backend.models.auth.scopes_default, help_text='The scopes we use when requesting user infos')),
                 ('userinfo_field_groups', models.CharField(blank=True, default='', help_text="The field in the userinfo (served by this IdP) that contains groups of the user.\n        Leave blank if you don't want to use groups of this IdP", max_length=200)),
                 ('userinfo_field_entitlements', models.CharField(blank=True, default='', help_text="The field in the userinfo (served by this IdP) that contains entitlements of the user.\n        Leave blank if you don't want to use entitlements of this IdP", max_length=200)),
             ],

feudal/backend/migrations/0012_auto_20200124_1403.py

@@ -2,7 +2,7 @@
 
 import django.core.validators
 from django.db import migrations, models
-import feudal.backend.auth.v1.models
+import feudal.backend.models.auth
 
 
 class Migration(migrations.Migration):
@@ -20,6 +20,6 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='oidcconfig',
             name='name',
-            field=models.CharField(max_length=200, validators=[feudal.backend.auth.v1.models.IDPNameValidator()]),
+            field=models.CharField(max_length=200, validators=[feudal.backend.models.auth.IDPNameValidator()]),
         ),
     ]

feudal/backend/migrations/0014_auto_20200213_1129.py

+# Generated by Django 3.0.3 on 2020-02-13 10:29
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('backend', '0013_auto_20200130_1528'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='credentialstate',
+            old_name='target',
+            new_name='deployment_state',
+        ),
+        migrations.AlterField(
+            model_name='deployment',
+            name='user',
+            field=models.ForeignKey(default=1, on_delete=django.db.models.deletion.CASCADE, related_name='deployments', to=settings.AUTH_USER_MODEL),
+            preserve_default=False,
+        ),
+    ]

feudal/backend/models/__init__.py

@@ -3,10 +3,8 @@ from logging import getLogger
 
 from django.db import models
 
-# these imports are exports!
-from ..auth.v1.models.vo import VO
-from .brokers import RabbitMQInstance
-from .users import User, SSHPublicKey
+from feudal.backend.models.users import User
+from feudal.backend.models.auth.vos import VO
 
 LOGGER = getLogger(__name__)
 
@@ -78,36 +76,25 @@ class Service(models.Model):
     # default values or values to update are expected as kwargs
     @classmethod
     def get_service(cls, name, site, **kwargs):
-        description = kwargs.get('description', '')
-        contact_email = kwargs.get('contact_email', '')
-        contact_description = kwargs.get('contact_description', '')
-        vos = kwargs.get('vos', None)
+        # non essential args
+        defaults = {}
+        for key in ['description', 'contact_email', 'contact_description']:
+            if key in kwargs:
+                defaults[key] = kwargs[key]
 
-        service, created = cls.objects.get_or_create(
+        service, created = cls.objects.update_or_create(
             name=name,
             site=site,
-            defaults={
-                'description': description,
-                'contact_email': contact_email,
-                'contact_description': contact_description,
-            },
+            defaults=defaults,
         )
         if created:
-            service.save()
-            if vos is not None:
-                for vo in vos:
-                    service.vos.add(vo)
+            LOGGER.info('Service created: %s', service)
 
-        else:
-            if service.description != description:
-                service.description = description
-                service.save()
-            if service.contact_email != contact_email:
-                service.contact_email = contact_email
-                service.save()
-            if service.contact_description != contact_description:
-                service.contact_description = contact_description
-                service.save()
+        if 'vos' in kwargs:
+            for vo in kwargs['vos']:
+                if not service.vos.contains(vo):
+                    LOGGER.info('VO %s now permits use of service %s', vo, service)
+                    service.vos.add(vo)
 
         return service
 

feudal/backend/auth/v1/models/__init__.py → feudal/backend/models/auth/__init__.py

@@ -112,6 +112,9 @@ class OIDCConfig(db_models.Model):
         return self.oidc_client.provider_info
 
     def __str__(self):
+        if not self.enabled:
+            return '{} (disabled)'.format(self.name)
+
         return self.name
 
     def get_auth_request(self, state, nonce):

feudal/backend/auth/v1/models/serializers/__init__.py → feudal/backend/models/auth/serializers/__init__.py

@@ -4,8 +4,8 @@
 from rest_framework import serializers
 from rest_polymorphic.serializers import PolymorphicSerializer
 
-from .. import OIDCConfig
-from ..vo import VO, Group, Entitlement, EntitlementNameSpace
+from feudal.backend.models.auth import OIDCConfig
+from feudal.backend.models.auth.vos import VO, Group, Entitlement, EntitlementNameSpace
 
 
 class OIDCConfigSerializer(serializers.ModelSerializer):

feudal/backend/auth/v1/models/serializers/clients.py → feudal/backend/models/auth/serializers/clients.py

@@ -2,7 +2,7 @@
 from rest_framework import serializers
 from rest_polymorphic.serializers import PolymorphicSerializer
 
-from ..vo import VO, Group, Entitlement, EntitlementNameSpace
+from feudal.backend.models.auth.vos import VO, Group, Entitlement, EntitlementNameSpace
 
 
 class EntitlementNameSpaceSerializer(serializers.ModelSerializer):

feudal/backend/auth/v1/models/test_models.py → feudal/backend/models/auth/test_models.py

 
 import logging
+
 from django.core.exceptions import ImproperlyConfigured
 from django.test import TestCase
 
-from feudal.backend.auth.v1.models import default_idp
+from feudal.backend.models.auth import default_idp
 
 LOGGER = logging.getLogger(__name__)
 

feudal/backend/auth/v1/models/vo.py → feudal/backend/models/auth/vos.py

@@ -5,7 +5,7 @@ import re
 from polymorphic.models import PolymorphicModel
 from django.db import models
 
-from . import OIDCConfig
+from feudal.backend.models.auth import OIDCConfig
 
 LOGGER = logging.getLogger(__name__)
 

feudal/backend/models/brokers.py

 
-# pylint: disable=import-outside-toplevel
-
 from logging import getLogger
 from json import dumps
 

feudal/backend/models/deployments.py

@@ -6,13 +6,14 @@ from django.db import models
 from django.db.models import Q
 from django.db.models.signals import post_delete
 from django.dispatch import receiver
+
 from django_mysql.models import JSONField
 from polymorphic.models import PolymorphicModel
 
-from feudal.backend.auth.v1.models.vo import VO
-
-from . import Site, Service, brokers
-from .users import User, SSHPublicKey
+from feudal.backend.models import Site, Service
+from feudal.backend.models.brokers import publish_to_user, RabbitMQInstance
+from feudal.backend.models.users import User, SSHPublicKey
+from feudal.backend.models.auth.vos import VO
 
 
 LOGGER = getLogger(__name__)
@@ -75,9 +76,8 @@ class Deployment(PolymorphicModel):
     user = models.ForeignKey(
         User,
         related_name='deployments',
-        # TODO check what repercussions the change to set null has
-        on_delete=models.SET_NULL,
-        null=True,
+        # deleting all deployments causes the DeploymentState to be orphaned -> removes deployments from the sites
+        on_delete=models.CASCADE,
     )
 
     # which state do we currently want to reach?
@@ -127,8 +127,7 @@ class Deployment(PolymorphicModel):
         self.publish_to_user()
 
     def _assure_states_exist(self):
-        # overridden in the specific implementations
-        pass
+        raise NotImplementedError('Should be overwritten in subclass')
 
     def update(self):
         self._assure_states_exist()
@@ -136,9 +135,6 @@ class Deployment(PolymorphicModel):
 
     # call when you changed Deployment.state_target
     def target_changed(self):
-        # save this instance to be save ;)
-        self.save()
-
         LOGGER.debug(self.msg('target_changed: {}'.format(self.state_target)))
 
         self._assure_states_exist()
@@ -157,7 +153,6 @@ class Deployment(PolymorphicModel):
         for item in self.states.all():
             item.user_credential_added(key)
 
-        # TODO is this publishing needed?
         if self.state_target == DEPLOYED:
             self.publish()
 
@@ -165,7 +160,6 @@ class Deployment(PolymorphicModel):
         for item in self.states.all():
             item.user_credential_removed(key)
 
-        # TODO is this publishing needed?
         if self.state_target == DEPLOYED:
             self.publish()
 
@@ -175,20 +169,16 @@ class Deployment(PolymorphicModel):
 
     # sends a state update via RabbitMQ / STOMP to the users webpage instance
     def publish_to_user(self):
-        if self.user is None:
-            return
-
         if settings.DEBUG_PUBLISHING:
             LOGGER.debug(self.msg('publish_to_user: {}'.format(self.state_target)))
 
-        brokers.publish_to_user(
+        publish_to_user(
             self.user,
             {
                 'deployment': self,
             },
         )
 
-    # TODO is it good to publish from here to the client?
     def publish(self):
         self.publish_to_user()
         self.publish_to_client()
@@ -222,37 +212,38 @@ class VODeployment(Deployment):
     def routing_key(self):
         return self.vo.name
 
+    # _assure_states_exist creates missing DeploymentState for this deployment
+    # returns True if new states were created
     def _assure_states_exist(self):
+        created_new_states = False
+
         for service in self.services:
-            DeploymentState.get_state_item(
-                self.user,
-                service.site,
-                service,
-                deployments=[self],
+            state, created = DeploymentState.objects.get_or_create(
+                user=self.user,
+                site=service.site,
+                service=service,
             )
+            if created:
+                LOGGER.debug(state.msg('Created'))
+                state.bind_to_deployment(self)  # bind new states to us
+
+            created_new_states = created
+
+        return created_new_states
 
     @classmethod
     def get_deployment(cls, user, vo):
-        try:
-            deployment = cls.objects.get(
-                user=user,
-                vo=vo,
-            )
-            deployment.update()
-
-            return deployment
+        deployment, created = cls.objects.get_or_create(
+            user=user,
+            vo=vo,
+        )
 
-        except cls.DoesNotExist:
-            deployment = cls(
-                user=user,
-                vo=vo,
-            )
+        if created:
+            LOGGER.debug(deployment.msg('Created'))
 
-            deployment.save()
-            deployment.update()
+            deployment.update()  # creates deployment states and publishes the creation
 
-            LOGGER.debug(deployment.msg('Created'))
-            return deployment
+        return deployment
 
     def service_added(self, service):
         LOGGER.debug(self.msg('Adding service {}'.format(service)))
@@ -302,6 +293,12 @@ class ServiceDeployment(Deployment):
 
     @classmethod
     def get_deployment(cls, user, service):
+        deployment, created = cls.object.get_or_create(user=user, service=service)
+        if created:
+            LOGGER.debug(deployment.msg('Created'))
+
+        deployment.update()
+
         try:
             deployment = cls.objects.get(
                 user=user,
@@ -319,7 +316,6 @@ class ServiceDeployment(Deployment):
             deployment.save()
             deployment.update()
 
-            LOGGER.debug(deployment.msg('Created'))
             return deployment
 
     def msg(self, msg):
@@ -364,7 +360,7 @@ class DeploymentState(models.Model):
         on_delete=models.CASCADE,
     )
 
-    # FIXME deleting a service currently deletes all deployment states without removing deployments!
+    # Deleting a service deletes all deployment states without removing deployments!
     service = models.ForeignKey(
         Service,
         related_name='states',
@@ -405,20 +401,21 @@ class DeploymentState(models.Model):
 
     @property
     def state_target(self):
-        # this instance needs to be saved before accessing a many to many field
-        if self.pk is not None:
-            for deployment in self.deployments.all():
-                if deployment.state_target == DEPLOYED:
-                    return DEPLOYED
+        for deployment in self.deployments.all():
+            if deployment.state_target == DEPLOYED:
+                return DEPLOYED
 
         return NOT_DEPLOYED
 
     @property
     def is_orphaned(self):
-        return not self.deployments.exists()
+        return not self.deployments.exists() or self.user is None
 
     @property
     def is_pending(self):
+        if self.user is None:
+            return True  # pending until deleted
+
         # When in these states we are never pending
         if self.state in [QUESTIONNAIRE, REJECTED]:
             return False
@@ -437,58 +434,53 @@ class DeploymentState(models.Model):
 
         return False
 
-    # TODO not accessible when the user is deleted
     @property
     def user_credentials(self):
+        if self.user is None:
+            LOGGER.debug(self.msg('User is None'))
+            raise User.DoesNotExist
+
         return self.user.credentials
 
     # get_state_item returns a state item for a given user, site and service
     # if the state does not exist it is created
     @classmethod
     def get_state_item(cls, user, site, service, deployments=None):
-        try:
-            item = cls.objects.get(
-                user=user,
-                site=site,
-                service=service,
-            )
-
-            # connect state item to the provided deployments
-            if deployments is not None:
-                for deployment in deployments:
-                    if not item.deployments.filter(id=deployment.id).exists():
-                        LOGGER.debug(item.msg('Binding to deployment {}'.format(deployment)))
-                        item.deployments.add(deployment)
+        item, created = cls.objects.get_or_create(
+            user=user,
+            site=site,
+            service=service,
+        )
 
-            return item
 
-        except cls.DoesNotExist:
-            item = cls(
-                user=user,
-                site=site,
-                service=service,
-            )
-            item.save()
-
-            if deployments is not None:
-                for deployment in deployments:
+        # connect state item to the provided deployments
+        if deployments is not None:
+            for deployment in deployments:
+                if not item.deployments.filter(id=deployment.id).exists():
+                    LOGGER.debug(item.msg('Binding to deployment {}'.format(deployment)))
                     item.deployments.add(deployment)
 
+        if created:
             LOGGER.debug(item.msg('Created'))
-
             item.dep_target_changed()
 
-            return item
+        return item
+
+    # adds the deployment to our related deployments
+    def bind_to_deployment(self, deployment):
+        if not self.deployments.filter(id=deployment.id).exists():
+            LO