Commit 48ca3b7f authored by Lukas Burgey's avatar Lukas Burgey
Browse files

Rework service configuration

Closes #3
parent 42d23327
...@@ -29,15 +29,6 @@ type ( ...@@ -29,15 +29,6 @@ type (
} }
) )
func appendIfMissing(slice []string, newElem string) []string {
for _, elem := range slice {
if elem == newElem {
return slice
}
}
return append(slice, newElem)
}
func (c config) Log(formatString string, params ...interface{}) { func (c config) Log(formatString string, params ...interface{}) {
log.Printf("%s "+formatString, append([]interface{}{"[Conf]"}, params...)...) log.Printf("%s "+formatString, append([]interface{}{"[Conf]"}, params...)...)
} }
...@@ -48,31 +39,32 @@ func (c consumer) Log(formatString string, params ...interface{}) { ...@@ -48,31 +39,32 @@ func (c consumer) Log(formatString string, params ...interface{}) {
func (c *config) consumer() (cons *consumer) { func (c *config) consumer() (cons *consumer) {
// calculate all the routing keys for the exchanges // calculate all the routing keys for the exchanges
var (
groupRoutingKeys = make([]string, len(c.GroupToServiceIDs))
entitlementRoutingKeys = make([]string, len(c.EntitlementToServiceIDs))
serviceRoutingKeys = make([]string, len(c.Services))
)
groupRoutingKeys := []string{} i := 0
entitlementRoutingKeys := []string{} for groupName := range c.GroupToServiceIDs {
groupRoutingKeys[i] = groupName
// all services we provide (either for groups or entitlements) i++
serviceRoutingKeys := []string{}
for groupName, groupServices := range c.GroupToServices {
groupRoutingKeys = appendIfMissing(groupRoutingKeys, groupName)
for _, groupService := range groupServices {
serviceRoutingKeys = appendIfMissing(serviceRoutingKeys, groupService.Name)
}
} }
for entitlementName, entitlementServices := range c.EntitlementToServices { i = 0
entitlementRoutingKeys = appendIfMissing(entitlementRoutingKeys, entitlementName) for entitlementName := range c.EntitlementToServiceIDs {
entitlementRoutingKeys[i] = entitlementName
i++
}
for _, entitlementService := range entitlementServices { i = 0
serviceRoutingKeys = appendIfMissing(serviceRoutingKeys, entitlementService.Name) for _, service := range c.Services {
} serviceRoutingKeys[i] = service.Name
i++
} }
cons = &consumer{ cons = &consumer{
uri: fmt.Sprintf("amqps://%s:%s@%s", c.Username, c.Password, c.Host), uri: fmt.Sprintf("amqps://%s:%s@%s", c.Username, c.Password, c.Hostname),
exchanges: []exchange{ exchanges: []exchange{
exchange{ exchange{
"groups", "topic", groupRoutingKeys, "groups", "topic", groupRoutingKeys,
...@@ -142,7 +134,7 @@ func (c *consumer) connect() (deliveries <-chan amqp.Delivery, err error) { ...@@ -142,7 +134,7 @@ func (c *consumer) connect() (deliveries <-chan amqp.Delivery, err error) {
false, // no-wait false, // no-wait
nil, nil,
); err != nil { ); err != nil {
err = fmt.Errorf("Error binding %s: %s", bindingKey, err) err = fmt.Errorf("Error binding %s:\n\t%s", bindingKey, err)
return return
} }
} }
...@@ -205,7 +197,7 @@ func (c *consumer) reconnect() { ...@@ -205,7 +197,7 @@ func (c *consumer) reconnect() {
func (c *consumer) startConsuming() (err error) { func (c *consumer) startConsuming() (err error) {
deliveries, err := c.connect() deliveries, err := c.connect()
if err != nil { if err != nil {
log.Printf("[AMQP] Error connecting: %s", err) err = fmt.Errorf("[AMQP] Error connecting: %s", err)
return return
} }
......
...@@ -2,6 +2,9 @@ module git.scc.kit.edu/feudal/feudalClient ...@@ -2,6 +2,9 @@ module git.scc.kit.edu/feudal/feudalClient
require ( require (
git.scc.kit.edu/feudal/feudalScripts v1.0.1 git.scc.kit.edu/feudal/feudalScripts v1.0.1
github.com/koron/iferr v0.0.0-20180615142939-bb332a3b1d91 // indirect
github.com/stamblerre/gocode v0.0.0-20181128172141-22843d89bc5a // indirect
github.com/streadway/amqp v0.0.0-20181107104731-27835f1a64e9 github.com/streadway/amqp v0.0.0-20181107104731-27835f1a64e9
golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce // indirect
gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/alecthomas/kingpin.v2 v2.2.6
) )
...@@ -7,7 +7,13 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5Vpd ...@@ -7,7 +7,13 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5Vpd
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/koron/iferr v0.0.0-20180615142939-bb332a3b1d91 h1:hunjgdb3b21ZdRmzDPXii0EcnHpjH7uCP+kODoE1JH0=
github.com/koron/iferr v0.0.0-20180615142939-bb332a3b1d91/go.mod h1:C2tFh8w3I6i4lnUJfoBx2Hwku3mgu4wPNTtUNp1i5KI=
github.com/stamblerre/gocode v0.0.0-20181128172141-22843d89bc5a h1:XVxDNb6jzFAgDYoLAazPpGEe+KBtjc/gLRPcC7taWEw=
github.com/stamblerre/gocode v0.0.0-20181128172141-22843d89bc5a/go.mod h1:EM2T8YDoTCvGXbEpFHxarbpv7VE26QD1++Cb1Pbh7Gs=
github.com/streadway/amqp v0.0.0-20181107104731-27835f1a64e9 h1:xBuwuVDG/vbGv1b0Dn/06flcq0R6MITax8244EZYaKE= github.com/streadway/amqp v0.0.0-20181107104731-27835f1a64e9 h1:xBuwuVDG/vbGv1b0Dn/06flcq0R6MITax8244EZYaKE=
github.com/streadway/amqp v0.0.0-20181107104731-27835f1a64e9/go.mod h1:1WNBiOZtZQLpVAyu0iTduoJL9hEsMloAK5XWrtW0xdY= github.com/streadway/amqp v0.0.0-20181107104731-27835f1a64e9/go.mod h1:1WNBiOZtZQLpVAyu0iTduoJL9hEsMloAK5XWrtW0xdY=
golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce h1:Gi26mRaGtAreZ9IadlBiwSJT1EDsfk4BSHBD9oxXEFY=
golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
...@@ -8,6 +8,7 @@ import ( ...@@ -8,6 +8,7 @@ import (
"log" "log"
"net/http" "net/http"
"os" "os"
"regexp"
"time" "time"
"gopkg.in/alecthomas/kingpin.v2" "gopkg.in/alecthomas/kingpin.v2"
...@@ -24,36 +25,51 @@ type ( ...@@ -24,36 +25,51 @@ type (
Site string `json:"site"` Site string `json:"site"`
} }
serviceID string
config struct { config struct {
Host string `json:"host"` Hostname string `json:"feudal_backend_host"`
Username string `json:"username"` Username string `json:"username"`
Password string `json:"password"` Password string `json:"password"`
// GroupToServices maps a group name to services provided for this group // Services maps an (arbitrary) service identifier to service structs
// this is for deployment per _group_ // the service identifiers are referenced in GroupToServiceIDs and EntitlementToServiceIDs
GroupToServices map[string]([]service) `json:"group_to_services"` Services map[serviceID]service
// EntitlementToServices maps a entitlement to services provided for users with this // GroupToServiceIDs determines which services are provided for users of the
// group
// maps a group name to service identifiers of the services
// services are declared in config.Services
GroupToServiceIDs map[string][]serviceID `json:"group_to_service_ids"`
// EntitlementToServiceIDs determines which services are provided for users of the
// entitlement // entitlement
EntitlementToServices map[string]([]service) `json:"entitlement_to_services"` // maps an entitlement to service identifiers of the services
// services are declared in config.Services
EntitlementToServiceIDs map[string][]serviceID `json:"entitlement_to_service_ids"`
// FetchIntervalString gets parsed by time.ParseDuration // FetchIntervalString gets parsed by time.ParseDuration
FetchIntervalString string `json:"fetch_interval"` FetchIntervalString string `json:"fetch_interval"`
// ReconnectTimeout gets parsed by time.ParseDuration // ReconnectTimeout gets parsed by time.ParseDuration
ReconnectTimeoutString string `json:"reconnect_timeout"` ReconnectTimeoutString string `json:"reconnect_timeout"`
NewTasks chan task
DoneTasks chan taskReply
FetchInterval time.Duration FetchInterval time.Duration
ReconnectTimeout time.Duration ReconnectTimeout time.Duration
RabbitMQConfig rabbitMQConfig RabbitMQConfig rabbitMQConfig
Site string Site string
NewTasks chan task
DoneTasks chan taskReply
Fetch chan struct{}
Error chan error
} }
// strippedConfig is sent to the backend on startup // strippedConfig is sent to the backend on startup
strippedConfig struct { strippedConfig struct {
GroupToServices map[string]([]service) `json:"group_to_services"` Services map[serviceID]service `json:"services"`
EntitlementToServices map[string]([]service) `json:"entitlement_to_services"` GroupToServiceIDs map[string][]serviceID `json:"group_to_service_ids"`
EntitlementToServiceIDs map[string][]serviceID `json:"entitlement_to_service_ids"`
} }
) )
...@@ -69,8 +85,6 @@ var ( ...@@ -69,8 +85,6 @@ var (
"Client for the Federated User Credential Deployment Portal (FEUDAL)", "Client for the Federated User Credential Deployment Portal (FEUDAL)",
).Author( ).Author(
"Lukas Burgey", "Lukas Burgey",
).Version(
"0.4.0",
) )
cmdStart = app.Command("start", "Starts the client in its normal operation mode.").Default() cmdStart = app.Command("start", "Starts the client in its normal operation mode.").Default()
cmdDeregister = app.Command("deregister", "Before disabling the client: Use deregister to inform the backend that the client ceases operation.") cmdDeregister = app.Command("deregister", "Before disabling the client: Use deregister to inform the backend that the client ceases operation.")
...@@ -90,7 +104,7 @@ func logError(err error, msg string) { ...@@ -90,7 +104,7 @@ func logError(err error, msg string) {
func (c *config) syncConfig() (err error) { func (c *config) syncConfig() (err error) {
log.Printf("[Conf] Synchronising configuration with %v", c.Host) log.Printf("[Conf] Synchronising configuration with %v", c.Hostname)
var ( var (
strippedConfigBytes []byte strippedConfigBytes []byte
...@@ -101,20 +115,23 @@ func (c *config) syncConfig() (err error) { ...@@ -101,20 +115,23 @@ func (c *config) syncConfig() (err error) {
// we inform the backend which services we provide // we inform the backend which services we provide
strippedConfigBytes, err = json.Marshal(strippedConfig{ strippedConfigBytes, err = json.Marshal(strippedConfig{
GroupToServices: c.GroupToServices, Services: c.Services,
EntitlementToServices: c.EntitlementToServices, GroupToServiceIDs: c.GroupToServiceIDs,
EntitlementToServiceIDs: c.EntitlementToServiceIDs,
}) })
if err != nil { if err != nil {
err = fmt.Errorf("Error syncing config: %s", err)
return return
} }
// update the services tracked by the backend // update the services tracked by the backend
req, err = http.NewRequest( req, err = http.NewRequest(
"PUT", "PUT",
"https://"+c.Host+"/backend/clientapi/config", "https://"+c.Hostname+"/backend/clientapi/config",
bytes.NewReader(strippedConfigBytes), bytes.NewReader(strippedConfigBytes),
) )
if err != nil { if err != nil {
err = fmt.Errorf("Error requesting remote config: %s", err)
return return
} }
...@@ -128,53 +145,48 @@ func (c *config) syncConfig() (err error) { ...@@ -128,53 +145,48 @@ func (c *config) syncConfig() (err error) {
body, err := ioutil.ReadAll(resp.Body) body, err := ioutil.ReadAll(resp.Body)
if err != nil { if err != nil {
err = fmt.Errorf("Error reading remote config: %s", err)
return return
} }
if resp.StatusCode != 200 { if resp.StatusCode != 200 {
err = fmt.Errorf("Unable to sync configuration (response: %v)", resp.Status) err = fmt.Errorf("Error with remote config request: Response was %v", resp.Status)
return return
} }
err = json.Unmarshal(body, &fetchedConfig) err = json.Unmarshal(body, &fetchedConfig)
if err != nil { if err != nil {
err = fmt.Errorf("Unable to parse remote configuration: %s %s", err, body) err = fmt.Errorf("Error parsing remote configuration: %s %s", err, body)
return return
} }
c.RabbitMQConfig = fetchedConfig.RabbitMQConfig c.RabbitMQConfig = fetchedConfig.RabbitMQConfig
c.Site = fetchedConfig.Site c.Site = fetchedConfig.Site
log.Printf("[Conf] Synchronised configuration with %v", c.Host) // initialize the task queues
c.NewTasks = make(chan task)
c.DoneTasks = make(chan taskReply)
c.Fetch = make(chan struct{})
c.Error = make(chan error)
log.Printf("[Conf] Synchronised configuration with %v", c.Hostname)
return return
} }
func getConfig(configFile string) (c config, err error) { func (c *config) validateConfig() (err error) {
c.Log("Reading config file %s", configFile)
bs, err := ioutil.ReadFile(configFile)
if err != nil {
c.Log("Error reading config file: %s", err)
return
}
err = json.Unmarshal(bs, &c)
if err != nil {
c.Log("Error parsing config file: %s", err)
return
}
// check the config values // check the config values
if c.Host == "" { if c.Hostname == "" {
log.Fatalf("[Conf] No 'host' in config") return fmt.Errorf("No 'hostname' in config")
} }
if c.Username == "" { if c.Username == "" {
log.Fatalf("[Conf] No 'user' in config") return fmt.Errorf("No 'username' in config")
} }
if c.Password == "" { if c.Password == "" {
log.Fatalf("[Conf] No 'password' in config") return fmt.Errorf("No 'password' in config")
} }
// try to parse duration, otherwise use default
if c.FetchIntervalString == "" { if c.FetchIntervalString == "" {
c.FetchInterval = defaultFetchInterval c.FetchInterval = defaultFetchInterval
log.Printf("[Conf] Using default fetch_interval of %v", c.FetchInterval) log.Printf("[Conf] Using default fetch_interval of %v", c.FetchInterval)
...@@ -203,12 +215,68 @@ func getConfig(configFile string) (c config, err error) { ...@@ -203,12 +215,68 @@ func getConfig(configFile string) (c config, err error) {
} }
} }
c.Log("Groups: %s", c.GroupToServices) // check that the services names are unique
c.Log("Entitlements: %s", c.EntitlementToServices) var exists bool
serviceNames := make(map[string]struct{}, len(c.Services))
for _, service := range c.Services {
if _, exists = serviceNames[service.Name]; !exists {
serviceNames[service.Name] = struct{}{}
} else {
return fmt.Errorf("Service name is ambiguous: %s", service.Name)
}
}
// initialize the task queues // check if referenced service ids exist
c.NewTasks = make(chan task) for _, sids := range c.GroupToServiceIDs {
c.DoneTasks = make(chan taskReply) for _, sid := range sids {
if _, exists = c.Services[sid]; !exists {
return fmt.Errorf("GroupToServiceIDs: service ID '%s' does not exist", sid)
}
}
}
for _, sids := range c.EntitlementToServiceIDs {
for _, sid := range sids {
if _, exists = c.Services[sid]; !exists {
return fmt.Errorf("EntitlementToServiceIDs: service ID '%s' does not exist", sid)
}
}
}
return
}
func getConfig(configFile string) (c config, err error) {
c.Log("Reading config file %s", configFile)
configBytes, err := ioutil.ReadFile(configFile)
if err != nil {
return
}
err = json.Unmarshal(configBytes, &c)
if err != nil {
return
}
err = c.validateConfig()
if err != nil {
err = fmt.Errorf("Error validating config:\n\t%s", err)
return
}
// strip the group authority from entitlement names
nameExtractor := regexp.MustCompile("^(.*?)#")
for entName, entServices := range c.EntitlementToServiceIDs {
match := nameExtractor.FindStringSubmatch(entName)
if len(match) == 2 {
delete(c.EntitlementToServiceIDs, entName)
c.EntitlementToServiceIDs[match[1]] = entServices
}
}
c.Log("Services: %s", c.Services)
c.Log("Groups: %s", c.GroupToServiceIDs)
c.Log("Entitlements: %s", c.EntitlementToServiceIDs)
return return
} }
...@@ -223,7 +291,7 @@ func (c *config) deregister() { ...@@ -223,7 +291,7 @@ func (c *config) deregister() {
req, err = http.NewRequest( req, err = http.NewRequest(
"PUT", "PUT",
"https://"+c.Host+"/backend/clientapi/deregister", "https://"+c.Hostname+"/backend/clientapi/deregister",
nil, nil,
) )
if err != nil { if err != nil {
...@@ -244,27 +312,31 @@ func (c *config) deregister() { ...@@ -244,27 +312,31 @@ func (c *config) deregister() {
} }
func (c *config) start() { func (c *config) start() {
if len(c.EntitlementToServices) == 0 && len(c.GroupToServices) == 0 { if len(c.EntitlementToServiceIDs) == 0 && len(c.GroupToServiceIDs) == 0 {
log.Printf("[P] Not starting pubsub because the are no services to subscribe to") log.Printf("[P] Not starting pubsub because the are no services to subscribe to")
return return
} }
// start task handler and responder // start task handler and responder
go c.taskFetcher()
go c.taskHandler() go c.taskHandler()
go c.taskResponder() go c.taskResponder()
consumer := c.consumer() consumer := c.consumer()
defer consumer.close() defer consumer.close()
consumer.startConsuming() var err error
err = consumer.startConsuming()
// start the fetcher after the consuming starts if err == nil {
// -> we miss nothing // fetch (after the connection is opened
go c.taskFetcher() c.Fetch <- struct{}{}
// wait until an error occurs, log it and crash
err = <-c.Error
}
// run till killed log.Fatalf("Fatal: %s", err)
forever := make(chan bool)
<-forever
} }
func main() { func main() {
...@@ -290,7 +362,7 @@ func main() { ...@@ -290,7 +362,7 @@ func main() {
// read the config file // read the config file
c, err := getConfig(*configFile) c, err := getConfig(*configFile)
if err != nil { if err != nil {
log.Fatalf("[Exit] No valid config. Exiting") log.Fatalf("[Conf] %s", err)
} }
switch cmd { switch cmd {
......
...@@ -82,48 +82,52 @@ func (te taskReply) Log(formatString string, params ...interface{}) { ...@@ -82,48 +82,52 @@ func (te taskReply) Log(formatString string, params ...interface{}) {
log.Printf("%s "+formatString, append([]interface{}{te}, params...)...) log.Printf("%s "+formatString, append([]interface{}{te}, params...)...)
} }
// TODO move to main.go
func (c *config) getServices(sids []serviceID) (services []service, err error) {
var ok bool
services = make([]service, len(sids))
for i, sid := range sids {
if services[i], ok = c.Services[sid]; !ok {
services = nil
err = fmt.Errorf("Service with service ID '%s' does no exist", sid)
return
}
}
return
}
// taskServices finds the services which need to be deployed for the given task // taskServices finds the services which need to be deployed for the given task
func (c *config) taskServices(t task) (services []service, err error) { func (c *config) taskServices(t task) (services []service, err error) {
// Option 1: VODeployment // Option 1: VODeployment
if t.ResourceType == "VODeployment" && t.VO != (vo{}) { if t.ResourceType == "VODeployment" && t.VO != (vo{}) {
var ok bool
var sids []serviceID
if t.VO.ResourceType == "Group" { if t.VO.ResourceType == "Group" {
var ok bool sids, ok = c.GroupToServiceIDs[t.VO.Name]
services, ok = c.GroupToServices[t.VO.Name]
if !ok { if !ok {
err = fmt.Errorf( err = fmt.Errorf("%s: Group %s does not exist", t, t.VO.Name)
"%s: Group %s has no mapped services",
t,
t.VO.Name,
)
return return
} }
} else if t.VO.ResourceType == "Entitlement" { } else if t.VO.ResourceType == "Entitlement" {
var ok bool sids, ok = c.EntitlementToServiceIDs[t.VO.Name]
services, ok = c.EntitlementToServices[t.VO.Name]
if !ok { if !ok {
err = fmt.Errorf( err = fmt.Errorf("%s: Entitlemnet %s does not exist", t, t.VO.Name)
"%s: Entitlement %s has no mapped services",
t,
t.VO.Name,
)
return return
} }
} }
} else if t.ResourceType == "ServiceDeployment" && t.Service != (service{}) { services, err = c.getServices(sids)
// we should hopefully only find one matching service if err != nil {
// See Issue #3 return
for _, groupServices := range c.GroupToServices {
for _, service := range groupServices {
if service.Name == t.Service.Name {
services = append(services, service)
}
}