Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
feudal
feudalBackend
Commits
d84890f9
Commit
d84890f9
authored
Nov 28, 2018
by
Lukas Burgey
Browse files
Implement simultaneous VO and Service Deployments
Makes Deployment and DeploymentState many to many.
parent
5197f6c6
Changes
8
Hide whitespace changes
Inline
Side-by-side
feudal/backend/migrations/0028_auto_20181128_1400.py
0 → 100644
View file @
d84890f9
# Generated by Django 2.1.3 on 2018-11-28 13:00
from
django.conf
import
settings
from
django.db
import
migrations
,
models
import
django.db.models.deletion
class
Migration
(
migrations
.
Migration
):
dependencies
=
[
(
'backend'
,
'0027_auto_20181126_1314'
),
]
operations
=
[
migrations
.
RemoveField
(
model_name
=
'deploymentstate'
,
name
=
'parent'
,
),
migrations
.
AddField
(
model_name
=
'deploymentstate'
,
name
=
'deployments'
,
field
=
models
.
ManyToManyField
(
related_name
=
'states'
,
to
=
'backend.Deployment'
),
),
migrations
.
AlterField
(
model_name
=
'deploymentstate'
,
name
=
'service'
,
field
=
models
.
ForeignKey
(
on_delete
=
django
.
db
.
models
.
deletion
.
CASCADE
,
related_name
=
'states'
,
to
=
'backend.Service'
),
),
migrations
.
AlterField
(
model_name
=
'deploymentstate'
,
name
=
'site'
,
field
=
models
.
ForeignKey
(
on_delete
=
django
.
db
.
models
.
deletion
.
CASCADE
,
related_name
=
'states'
,
to
=
'backend.Site'
),
),
migrations
.
AlterField
(
model_name
=
'deploymentstate'
,
name
=
'user'
,
field
=
models
.
ForeignKey
(
blank
=
True
,
null
=
True
,
on_delete
=
django
.
db
.
models
.
deletion
.
SET_NULL
,
related_name
=
'states'
,
to
=
settings
.
AUTH_USER_MODEL
),
),
]
feudal/backend/models/__init__.py
View file @
d84890f9
...
...
@@ -43,15 +43,19 @@ class Site(models.Model):
# ignores orphaned deployment states
dep_states
=
[
state
for
state
in
self
.
state
_item
s
.
all
()
for
state
in
self
.
states
.
all
()
if
(
state
.
is_pending
or
state
.
is_credential_pending
)
and
state
.
parent
is
not
None
)
and
state
.
deployments
.
exists
()
]
# make the deployments unique here
for
item
in
dep_states
:
deployments
[
item
.
parent
.
id
]
=
item
.
parent
for
state
in
dep_states
:
# we filter for deployments of the same state_target here
# if we have multiple deployments per deployment state this causes only the
# deployments with the same state_target to be pending
for
deployment
in
state
.
deployments
.
filter
(
state_target
=
state
.
state_target
):
deployments
[
deployment
.
id
]
=
deployment
return
deployments
.
values
()
...
...
feudal/backend/models/deployments.py
View file @
d84890f9
...
...
@@ -4,6 +4,7 @@ from logging import getLogger
from
django.conf
import
settings
from
django.db
import
models
from
django.db.models
import
Q
from
django_mysql.models
import
JSONField
from
polymorphic.models
import
PolymorphicModel
...
...
@@ -86,9 +87,9 @@ class Deployment(PolymorphicModel):
@
property
def
state
(
self
):
if
self
.
state
_item
s
.
exists
():
if
self
.
states
.
exists
():
_state
=
''
for
state
in
self
.
state
_item
s
.
all
():
for
state
in
self
.
states
.
all
():
if
_state
==
''
:
_state
=
state
.
state
elif
_state
!=
state
.
state
:
...
...
@@ -104,44 +105,69 @@ class Deployment(PolymorphicModel):
return
self
.
state_target
==
self
.
state
def
_set_target
(
self
,
target
):
LOGGER
.
debug
(
self
.
msg
(
'Target changed to '
+
target
))
if
str
(
self
.
state_target
)
==
str
(
target
):
return
self
.
state_target
=
target
LOGGER
.
debug
(
self
.
msg
(
'Target: {} -> {} '
.
format
(
self
.
state_target
,
target
)))
for
credential_state
in
CredentialState
.
objects
.
filter
(
target__parent
=
self
,
):
credential_state
.
set_target
(
target
)
self
.
state_target
=
target
self
.
save
()
# FIXME this is breaking things when one wants deploy and another gets a user_remove
# # set the target to all credentials
# for state in self.states.all():
# state.assure_credential_states_exist()
# for credential_state in CredentialState.objects.filter(
# target__deployments=self,
# ):
# credential_state.set_target(target)
# Deployment.user_deploy
def
user_deploy
(
self
):
self
.
_set_target
(
'deployed'
)
for
item
in
self
.
state_items
.
all
():
LOGGER
.
debug
(
self
.
msg
(
'user_deploy'
))
self
.
_set_target
(
DEPLOYED
)
# states which are not DEPLOYED
for
item
in
self
.
states
.
filter
(
~
Q
(
state
=
DEPLOYED
)):
item
.
user_deploy
()
self
.
publish_to_client
()
# each state item publishes its state to the user
self
.
publish
()
# Deployment.user_remove
def
user_remove
(
self
):
for
item
in
self
.
state_items
.
all
():
if
item
.
state
!=
NOT_DEPLOYED
:
item
.
user_remove
()
LOGGER
.
debug
(
self
.
msg
(
'user_remove'
))
can_publish
=
False
self
.
_set_target
(
NOT_DEPLOYED
)
self
.
publish_to_client
()
# each state item publishes its state to the user
# states which are not NOT_DEPLOYED
for
item
in
self
.
states
.
filter
(
~
Q
(
state
=
NOT_DEPLOYED
)):
if
item
.
user_remove
():
can_publish
=
True
# we only publish to the clients if allowed
# we are not allowed to publish a removal if another deployment for our
# DeploymentStates exists and has the target DEPLOYED
if
can_publish
:
self
.
publish
()
else
:
self
.
publish_to_user
()
def
user_credential_added
(
self
,
key
):
for
item
in
self
.
state
_item
s
.
all
():
for
item
in
self
.
states
.
all
():
item
.
user_credential_added
(
key
)
self
.
publish
()
def
user_credential_removed
(
self
,
key
):
for
item
in
self
.
state
_item
s
.
all
():
for
item
in
self
.
states
.
all
():
item
.
user_credential_removed
(
key
)
self
.
publish
()
def
publish_to_client
(
self
):
LOGGER
.
debug
(
self
.
msg
(
'publish_to_client'
))
from
.serializers
import
clients
data
=
clients
.
DeploymentSerializer
(
self
).
data
msg
=
dumps
(
data
)
...
...
@@ -156,6 +182,8 @@ class Deployment(PolymorphicModel):
if
self
.
user
is
None
:
return
LOGGER
.
debug
(
self
.
msg
(
'publish_to_user'
))
from
.
import
serializers
msg
=
dumps
({
'deployment'
:
serializers
.
DeploymentSerializer
(
self
).
data
,
...
...
@@ -198,13 +226,13 @@ class VODeployment(Deployment):
def
routing_key
(
self
):
return
self
.
vo
.
name
def
create_state
_item
s
(
self
):
def
create_states
(
self
):
for
service
in
self
.
services
:
DeploymentState
.
get_state_item
(
self
,
self
.
user
,
service
.
site
,
service
,
deployments
=
[
self
],
)
@
classmethod
...
...
@@ -214,7 +242,7 @@ class VODeployment(Deployment):
user
=
user
,
vo
=
vo
,
)
deployment
.
create_state
_item
s
()
deployment
.
create_states
()
return
deployment
...
...
@@ -225,7 +253,7 @@ class VODeployment(Deployment):
)
deployment
.
save
()
deployment
.
create_state
_item
s
()
deployment
.
create_states
()
LOGGER
.
debug
(
deployment
.
msg
(
'Created'
))
return
deployment
...
...
@@ -233,12 +261,12 @@ class VODeployment(Deployment):
def
service_added
(
self
,
service
):
LOGGER
.
debug
(
self
.
msg
(
'Adding service {}'
.
format
(
service
)))
item
=
DeploymentState
.
get_state_item
(
self
,
self
.
user
,
service
.
site
,
service
,
deployments
=
[
self
],
)
if
self
.
state_target
==
'deployed'
:
if
str
(
self
.
state_target
)
==
'deployed'
:
item
.
user_deploy
()
def
service_removed
(
self
,
service
):
...
...
@@ -272,12 +300,11 @@ class ServiceDeployment(Deployment):
return
self
.
service
.
name
def
create_state_item
(
self
):
LOGGER
.
debug
(
'create_state_item: creating DeploymentState for service %s at site %s'
,
self
.
service
,
self
.
service
.
site
)
DeploymentState
.
get_state_item
(
self
,
self
.
user
,
self
.
service
.
site
,
self
.
service
,
deployments
=
[
self
],
)
@
classmethod
...
...
@@ -315,18 +342,14 @@ class ServiceDeployment(Deployment):
class
DeploymentState
(
models
.
Model
):
parentless
=
ValueError
(
'Tried to access parent of parentless deployment state'
)
parent
=
models
.
ForeignKey
(
deployments
=
models
.
ManyToManyField
(
Deployment
,
related_name
=
'state_items'
,
on_delete
=
models
.
SET_NULL
,
null
=
True
related_name
=
'states'
,
)
user
=
models
.
ForeignKey
(
User
,
related_name
=
'state
_item
s'
,
related_name
=
'states'
,
on_delete
=
models
.
SET_NULL
,
blank
=
True
,
null
=
True
,
...
...
@@ -334,13 +357,13 @@ class DeploymentState(models.Model):
site
=
models
.
ForeignKey
(
Site
,
related_name
=
'state
_item
s'
,
related_name
=
'states'
,
on_delete
=
models
.
CASCADE
,
)
service
=
models
.
ForeignKey
(
Service
,
related_name
=
'state
_item
s'
,
related_name
=
'states'
,
on_delete
=
models
.
CASCADE
,
)
...
...
@@ -371,15 +394,23 @@ class DeploymentState(models.Model):
blank
=
True
,
)
@
property
def
state_target
(
self
):
for
deployment
in
self
.
deployments
.
all
():
if
deployment
.
state_target
==
DEPLOYED
:
return
DEPLOYED
return
NOT_DEPLOYED
@
property
def
is_pending
(
self
):
# TODO
# pending because we are orphaned -> pending until removed everywhere
if
self
.
parent
is
None
:
if
not
self
.
deployments
.
exists
()
:
return
True
# pending because the state target is not reached
if
self
.
parent
.
state_target
!=
self
.
state
:
if
self
.
state_target
!=
self
.
state
:
return
True
return
False
...
...
@@ -396,24 +427,30 @@ class DeploymentState(models.Model):
return
self
.
user
.
credentials
@
classmethod
def
get_state_item
(
cls
,
parent
,
user
,
site
,
service
):
def
get_state_item
(
cls
,
user
,
site
,
service
,
deployments
=
[]
):
try
:
item
=
cls
.
objects
.
get
(
parent
=
parent
,
user
=
user
,
site
=
site
,
service
=
service
,
)
for
deployment
in
deployments
:
if
not
item
.
deployments
.
filter
(
id
=
deployment
.
id
).
exists
():
LOGGER
.
debug
(
item
.
msg
(
'Binding to deployment {}'
.
format
(
deployment
)))
item
.
deployments
.
add
(
deployment
)
return
item
except
cls
.
DoesNotExist
:
item
=
cls
(
parent
=
parent
,
user
=
user
,
site
=
site
,
service
=
service
,
)
item
.
save
()
for
deployment
in
deployments
:
item
.
deployments
.
add
(
deployment
)
LOGGER
.
debug
(
item
.
msg
(
'Created'
))
return
item
...
...
@@ -441,14 +478,20 @@ class DeploymentState(models.Model):
# STATE TRANSITIONS
#
user: provisioning requested
#
DeploymentState.user_deploy
def
user_deploy
(
self
):
if
self
.
state
==
REMOVAL_PENDING
:
LOGGER
.
debug
(
self
.
msg
(
'user_deploy'
))
self
.
_assure_credential_states_exist
()
for
cred_state
in
self
.
credential_states
.
all
():
cred_state
.
set_target
(
DEPLOYED
)
if
str
(
self
.
state
)
==
REMOVAL_PENDING
:
self
.
_set_state
(
DEPLOYED
)
return
if
self
.
state
==
DEPLOYED
:
LOGGER
.
info
(
self
.
msg
(
'
ignoring invalid state transition user_
deploy'
))
if
str
(
self
.
state
)
==
DEPLOYED
:
LOGGER
.
debug
(
self
.
msg
(
'
State: already
deploy
ed
'
))
return
self
.
_set_state
(
...
...
@@ -456,46 +499,56 @@ class DeploymentState(models.Model):
publish
=
False
,
# the post response already contains the update
)
# user: deprovisioning requested
# DeploymentState.user_remove
# returns True if no other deployment needs this state_item to be deployed
def
user_remove
(
self
):
if
self
.
parent
is
None
:
LOGGER
.
error
(
'user_remove: parentless'
)
return
if
str
(
self
.
state_target
)
==
DEPLOYED
:
LOGGER
.
debug
(
self
.
msg
(
'user_remove: Not removing: another deployment has target deployed'
))
# False: signal the callee that a publish_to_client is *not* permitted
return
False
if
(
self
.
parent
.
state_target
==
DEPLOYED
and
(
self
.
state
==
FAILED
or
self
.
state
==
REJECTED
)
LOGGER
.
debug
(
self
.
msg
(
'user_remove'
))
for
cred_state
in
self
.
credential_states
.
all
():
cred_state
.
set_target
(
NOT_DEPLOYED
)
if
str
(
self
.
state
)
==
NOT_DEPLOYED
:
LOGGER
.
debug
(
self
.
msg
(
'State: already not_deployed'
))
elif
(
self
.
state_target
==
DEPLOYED
and
(
self
.
state
==
FAILED
or
self
.
state
==
REJECTED
)
):
self
.
_reset
()
self
.
_set_state
(
NOT_DEPLOYED
,
publish
=
False
)
return
if
self
.
state
==
NOT_DEPLOYED
:
LOGGER
.
info
(
self
.
msg
(
'ignoring invalid state transition user_remove'
))
return
# FIXME this will break if the client 'finishes' the deployment, after user_remove
if
(
elif
(
self
.
state
==
DEPLOYMENT_PENDING
or
self
.
state
==
QUESTIONNAIRE
):
self
.
_set_state
(
NOT_DEPLOYED
)
return
self
.
_set_state
(
REMOVAL_PENDING
,
publish
=
False
,
# the post response already contains the update
)
else
:
# default: start the removal process
self
.
_set_state
(
REMOVAL_PENDING
,
publish
=
False
,
# the post response already contains the update
)
# True: signal the callee that a publish_to_client is permitted
return
True
# user: questionnaire answered
def
user_answers
(
self
,
answers
=
None
):
if
self
.
parent
is
None
:
LOGGER
.
error
(
'user_
answers: parentles
s'
)
if
not
self
.
deployments
.
exists
()
:
LOGGER
.
error
(
'user_
remove: no deployment
s'
)
return
self
.
questionnaire
=
answers
self
.
_set_state
(
DEPLOYMENT_PENDING
,
publish
=
False
)
self
.
parent
.
publish_to_client
()
self
.
publish_to_client
()
def
client_credential_states
(
self
,
credential_states
):
# maps ssh key names to their state
...
...
@@ -512,9 +565,16 @@ class DeploymentState(models.Model):
else
:
credential_state
.
set
(
NOT_DEPLOYED
)
# returns None on success, or a string describing an error
#
client_response
returns None on success, or a string describing an error
def
client_response
(
self
,
output
):
state
=
output
.
get
(
'state'
,
None
)
if
'state'
not
in
output
:
return
'field "state" is missing in output'
state
=
output
.
get
(
'state'
,
''
)
LOGGER
.
debug
(
self
.
msg
(
'Client response: {}'
.
format
(
state
)))
self
.
_set_state
(
state
)
credential_states
=
output
.
get
(
'user_credential_states'
,
None
)
if
credential_states
is
not
None
:
...
...
@@ -523,27 +583,16 @@ class DeploymentState(models.Model):
self
.
message
=
output
.
get
(
'message'
,
''
)
self
.
save
()
if
state
is
None
:
return
'missing state in output'
# update values
if
state
==
DEPLOYED
:
self
.
credentials
=
output
.
get
(
'credentials'
,
{})
self
.
save
()
# the client completed a deployment after the user wished to remove the deployment
if
self
.
parent
is
not
None
and
self
.
parent
.
state_target
==
NOT_DEPLOYED
:
self
.
user_remove
()
elif
state
==
NOT_DEPLOYED
:
# reset credentials and questionnaire
self
.
_reset
()
self
.
save
()
# the client completed a removal after the user wished to deploy the deployment
if
self
.
parent
is
not
None
and
self
.
parent
.
state_target
==
DEPLOYED
:
self
.
user_deploy
()
elif
state
==
QUESTIONNAIRE
:
self
.
questionnaire
=
output
.
get
(
'questionnaire'
,
{})
self
.
save
()
...
...
@@ -552,9 +601,16 @@ class DeploymentState(models.Model):
elif
state
==
FAILED
:
pass
else
:
return
'unknown state
\'
'
+
state
+
'
\'
'
return
'unknown state "{}"'
.
format
(
state
)
# FIXME this apparently causes deployment loops
# is the target reached now?
if
self
.
state_target
!=
self
.
state
:
LOGGER
.
debug
(
self
.
msg
(
'Target {} still not reached. Publishing again'
.
format
(
self
.
state_target
,
)))
self
.
publish_to_client
()
self
.
_set_state
(
state
)
return
None
# resets all client sent values
...
...
@@ -563,7 +619,7 @@ class DeploymentState(models.Model):
self
.
questionnaire
=
questionnaire_default
()
self
.
message
=
''
def
_
set_state
(
self
,
state
,
publish
=
True
):
def
_
assure_credential_states_exist
(
self
):
# assure all user credentials have a state
if
self
.
user
is
not
None
:
for
key
in
self
.
user
.
ssh_keys
.
all
():
...
...
@@ -575,26 +631,38 @@ class DeploymentState(models.Model):
except
CredentialState
.
DoesNotExist
:
LOGGER
.
error
(
'CredentialState.DoesNotExist in _set_state'
)
if
(
state
==
DEPLOYMENT_PENDING
or
state
==
REMOVAL_PENDING
or
state
==
FAILED
):
self
.
pending
=
True
def
_set_state
(
self
,
state
,
publish
=
True
):
self
.
_assure_credential_states_exist
()
if
str
(
self
.
state
)
==
str
(
state
):
return
LOGGER
.
debug
(
self
.
msg
(
'State: {} -> {} - Target: {}'
.
format
(
self
.
state
,
state
,
self
.
state_target
)))
self
.
state
=
state
self
.
save
()
LOGGER
.
debug
(
self
.
msg
(
'State changed to '
+
self
.
state
))
if
publish
and
self
.
parent
is
not
None
:
self
.
parent
.
publish_to_user
()
# publish to user
if
publish
and
self
.
deployments
.
exists
():
self
.
publish_to_user
()
def
publish_to_user
(
self
):
for
deployment
in
self
.
deployments
.
all
():
deployment
.
publish_to_user
()
def
publish_to_client
(
self
):
for
deployment
in
self
.
deployments
.
all
():
deployment
.
publish_to_client
()
def
msg
(
self
,
msg
):
return
'{} - {}'
.
format
(
self
,
msg
)
def
__str__
(
self
):
if
self
.
parent
is
not
None
:
if
self
.
deployments
.
exists
():
deployment_names
=
[
str
(
deployment
.
id
)
for
deployment
in
self
.
deployments
.
all
()]
return
'DState: ({}:{}:{})#{}'
.
format
(
self
.
parent
.
id
,
','
.
join
(
deployment_names
)
,
self
.
service
,
self
.
site
,
self
.
id
,
...
...
@@ -630,6 +698,7 @@ class CredentialState(models.Model):
default
=
False
,
)
# TODO target is a stupid field name. Change it
target
=
models
.
ForeignKey
(
DeploymentState
,
related_name
=
'credential_states'
,
...
...
@@ -653,7 +722,7 @@ class CredentialState(models.Model):
credential
=
credential
,
target
=
target
,
state
=
NOT_DEPLOYED
,
state_target
=
target
.
parent
.
state_target
,
state_target
=
target
.
state_target
,
)
new_state
.
save
()
...
...
@@ -663,17 +732,22 @@ class CredentialState(models.Model):
return
new_state
def
set_target
(
self
,
target
):
if
str
(
self
.
state_target
)
==
str
(
target
):
return
# state_target is locked, since we are marked for deletion
if
self
.
_credential_deleted
:
return
LOGGER
.
debug
(
self
.
msg
(
'Target: {} -> {}'
.
format
(
self
.
state_target
,
target
)))
self
.
state_target
=
target
self
.
save
()
if
settings
.
DEBUG_CREDENTIALS
:
LOGGER
.
debug
(
self
.
msg
(
'Target changed to {}'
.
format
(
target
)))
def
set
(
self
,
state
):
if
str
(
self
.
state
)
==
str
(
state
):
return
if
state
==
NOT_DEPLOYED
and
self
.
_credential_deleted
:
self
.
_delete_state
()
return
...
...
@@ -681,10 +755,13 @@ class CredentialState(models.Model):
if
state
==
self
.
state
:
return
if
settings
.
DEBUG_CREDENTIALS
:
LOGGER
.
debug
(
self
.
msg
(
'State: {} -> {}'
.
format
(
self
.
state
,
state
)))
self
.
state
=
state