Commit 32b4d8aa authored by lukasburgey's avatar lukasburgey
Browse files

Merge branch 'dev'

parents 58fc6dea 83fb47f6
Pipeline #81670 failed with stages
in 3 minutes and 28 seconds
......@@ -2,3 +2,4 @@
deploy
example
configs
builds
image: golang:1.12-alpine
stages:
- lint
- build
- deploy
before_script:
- apk add --no-cache git make rsync openssh-client
- mkdir -p /go/src/git.scc.kit.edu/feudal /go/src/_/builds
- cp -r $CI_PROJECT_DIR /go/src/git.scc.kit.edu/feudal/feudalClient
- ln -s /go/src/git.scc.kit.edu/feudal /go/src/_/builds/feudalClient
- make dep
lint:
image: golang:1.12-alpine
stage: lint
script:
make lint
- apk add --no-cache git make
- make lint
build:
image: golang:1.12-alpine
stage: build
script:
- make build
- apk add --no-cache git make
- make build-static
artifacts:
paths:
- feudalClient
deploy_dev:
deploy-dev:
image: netroby/alpine-rsync
stage: deploy
environment:
name: dev
only:
- dev
- master
environment:
name: dev
dependencies:
- build
script:
- eval $(ssh-agent -s)
- echo "$SSH_PRIVATE_KEY" | ssh-add -
......@@ -39,13 +36,17 @@ deploy_dev:
- chmod 700 ~/.ssh
- echo "$SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts
- rsync -v feudalClient $DEV_USER@$DEV_HOST:bin/feudalClientDev
- ssh $DEV_USER@$DEV_HOST -- systemctl --user restart devClient@0 devClient@1 devClient@2
deploy_master:
deploy-master:
image: netroby/alpine-rsync
stage: deploy
environment:
name: master
only:
- master
environment:
name: master
dependencies:
- build
script:
- eval $(ssh-agent -s)
- echo "$SSH_PRIVATE_KEY" | ssh-add -
......
......@@ -3,25 +3,36 @@ PKG := "git.scc.kit.edu/feudal/$(PROJECT_NAME)"
PKG_LIST := $(shell go list ${PKG}/... | grep -v /vendor/)
GO_FILES := $(shell find . -name '*.go' | grep -v /vendor/ | grep -v _test.go)
.PHONY: all dep build clean test coverage coverhtml lint
.PHONY: all dep build clean test lint
all: build
lint: ## Lint the files
dep: ## Get the dependencies
@go get -v -d ./...
lint: dep ## Lint the files
@go get -u golang.org/x/lint/golint
@golint -set_exit_status ${PKG_LIST}
test: ## Run unittests
@go test -short ${PKG_LIST}
dep: ## Get the dependencies
@go get -v -d ./...
@go get -u golang.org/x/lint/golint
# link dynamically
build: dep ## Build the binary file
@go build -i -v $(PKG)
# link statically
build-static: dep ## Build the binary file
@CGO_ENABLED=0 GOOS=linux go build -v -ldflags '-extldflags "-static"' $(PKG)
clean: ## Remove previous build
@rm -f $(PROJECT_NAME)
help: ## Display this help screen
@grep -h -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
install: dep
@go install ./...
tidy:
@go mod tidy
......@@ -4,11 +4,15 @@ import (
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"regexp"
"strings"
"time"
"gopkg.in/yaml.v2"
)
type (
......@@ -17,84 +21,94 @@ type (
// Service for which we do deployments
Service struct {
Name string `json:"name"`
Command string `json:"command"`
Description string `json:"description"`
// Name is the name of the service. It is presented to users.
Name string `json:"name" yaml:"name"`
// Command is a shell command to invoke the adapter
Command string `json:"command" yaml:"command"`
// Description describes the service. It is presented to users
Description string `json:"description" yaml:"description"`
// ContactEmail is an email
ContactEmail string `json:"contact_email" yaml:"contact_email"`
// ContactDescription describes the person contable using ContactEmail
ContactDescription string `json:"contact_description" yaml:"contact_description"`
}
// RabbitMQConfig is used for the amqp source
RabbitMQConfig struct {
Exchanges []string `json:"exchanges"`
Vhost string `json:"vhost"`
Exchanges []string `json:"exchanges" yaml:"exchanges"`
Vhost string `json:"vhost" yaml:"vhost"`
}
fetchedConfig struct {
RabbitMQConfig RabbitMQConfig `json:"rabbitmq_config"`
Site string `json:"site"`
RabbitMQConfig RabbitMQConfig `json:"rabbitmq_config" yaml:"rabbitmq_config"`
Site string `json:"site" yaml:"site"`
}
// The DebugConfig sets the debuging levels for components of the client.
DebugConfig struct {
// Sequential causes the execution of only one deployment at a time for debugging purposes
Sequential bool `json:"sequential_execution"`
Scripts bool `json:"scripts"`
Backend bool `json:"backend"`
Sequential bool `json:"sequential_execution" yaml:"sequential_execution"`
Scripts bool `json:"scripts" yaml:"scripts"`
Backend bool `json:"backend" yaml:"backend"`
}
// Config is the structure of our config file
Config struct {
// The Hostname of the feudal backends host
Hostname string `json:"feudal_backend_host"`
Hostname string `json:"feudal_backend_host" yaml:"feudal_backend_host"`
// The Username of this client, registered at the backend.
Username string `json:"username"`
Username string `json:"username" yaml:"username"`
// The Password of this client, registered at the backend.
Password string `json:"password"`
Password string `json:"password" yaml:"password"`
// Services maps an (arbitrary) service identifier to service structs
// the service identifiers are referenced in GroupToServiceIDs and EntitlementToServiceIDs
Services map[ServiceID]Service `json:"services"`
Services map[ServiceID]Service `json:"services" yaml:"services"`
// GroupToServiceIDs determines which services are provided for users of the
// group
// maps a group name to service identifiers of the services
// services are declared in Config.Services
GroupToServiceIDs map[string][]ServiceID `json:"group_to_service_ids"`
GroupToServiceIDs map[string][]ServiceID `json:"group_to_service_ids" yaml:"group_to_service_ids"`
// EntitlementToServiceIDs determines which services are provided for users of the
// entitlement
// maps an entitlement to service identifiers of the services
// services are declared in Config.Services
EntitlementToServiceIDs map[string][]ServiceID `json:"entitlement_to_service_ids"`
EntitlementToServiceIDs map[string][]ServiceID `json:"entitlement_to_service_ids" yaml:"entitlement_to_service_ids"`
// FetchIntervalString gets parsed by time.ParseDuration
FetchIntervalString string `json:"fetch_interval,omitempty"`
FetchIntervalString string `json:"fetch_interval,omitempty" yaml:"fetch_interval,omitempty"`
// ReconnectTimeout gets parsed by time.ParseDuration
ReconnectTimeoutString string `json:"reconnect_timeout,omitempty"`
ReconnectTimeoutString string `json:"reconnect_timeout,omitempty" yaml:"reconnect_timeout,omitempty"`
// Debug flags control the log levels of the client.
Debug DebugConfig `json:"debug,omitempty"`
Debug DebugConfig `json:"debug,omitempty" yaml:"debug,omitempty"`
// After the duration of FetchInterval the client will fetch updates using the REST
// interface.
FetchInterval time.Duration `json:"-"`
FetchInterval time.Duration `json:"-" yaml:"-"`
// We reconnect to RabbitMQ after ReconnectTimeout.
ReconnectTimeout time.Duration `json:"-"`
ReconnectTimeout time.Duration `json:"-" yaml:"-"`
RabbitMQConfig RabbitMQConfig `json:"-"`
RabbitMQConfig RabbitMQConfig `json:"-" yaml:"-"`
// The Site is the name of the site of this client. It is fetched from the backend.
Site string `json:"-"`
Site string `json:"-" yaml:"-"`
}
// strippedConfig is sent to the backend on startup
strippedConfig struct {
Services map[ServiceID]Service `json:"services"`
GroupToServiceIDs map[string][]ServiceID `json:"group_to_service_ids"`
EntitlementToServiceIDs map[string][]ServiceID `json:"entitlement_to_service_ids"`
Services map[ServiceID]Service `json:"services" yaml:"services"`
GroupToServiceIDs map[string][]ServiceID `json:"group_to_service_ids" yaml:"group_to_service_ids"`
EntitlementToServiceIDs map[string][]ServiceID `json:"entitlement_to_service_ids" yaml:"entitlement_to_service_ids"`
}
)
......@@ -150,7 +164,7 @@ func (conf *Config) Sync() (err error) {
// update the services tracked by the backend
req, err = http.NewRequest(
"PUT",
"https://"+conf.Hostname+"/backend/clientapi/config",
"https://"+conf.Hostname+"/client/config",
bytes.NewReader(strippedConfigBytes),
)
if err != nil {
......@@ -262,20 +276,28 @@ func (conf *Config) validateConfig() (err error) {
}
// ReadConfig reads a config file and validates it
func ReadConfig(configFile string) (conf *Config, err error) {
if configFile == "" {
return nil, fmt.Errorf("Please provide the --config flag. See --help")
}
log.Printf("[Conf] Reading config file %s", configFile)
configBytes, err := ioutil.ReadFile(configFile)
func ReadConfig(r io.Reader, fileName string) (conf *Config, err error) {
configBytes, err := ioutil.ReadAll(r)
if err != nil {
return
}
conf = new(Config)
err = json.Unmarshal(configBytes, conf)
if err != nil {
if strings.HasSuffix(fileName, ".yaml") {
err = yaml.Unmarshal(configBytes, conf)
if err != nil {
err = fmt.Errorf("Error decoding yaml: %s", err)
return
}
} else if strings.HasSuffix(fileName, ".json") {
err = json.Unmarshal(configBytes, conf)
if err != nil {
err = fmt.Errorf("Error decoding json: %s", err)
return
}
} else {
err = fmt.Errorf("Incompatible config format, supported are: YAML or JSON")
return
}
......
{
"host" : "<host domain>",
"username" : "<username registerred at the backend>",
"password" : "<password registerred at the backend>",
"fetch_interval": "30m",
"reconnect_timeout": "10s",
"group_to_services": {
"<unity group name>": [
{
"name": "<service name>",
"command": "<executable command in PATH or absolute>",
"description": "<service description>"
}
"feudal_backend_host": "<foo>.kit.edu",
"username": "<username registerred at the portal>",
"password": "<password registerred at the portal>",
"debug": {
"backend": true,
"scripts": false,
"sequential_execution": false
},
"services": {
"<local identifier>": {
"name": "<service name>",
"description": "<service description>",
"command": "<full path to script/adapter>"
}
},
"group_to_service_ids": {
"/": [
"<local identifier>"
]
},
"entitlement_to_service_ids": {
"urn:geant:h-df.de:group:myExampleColab#unity.helmholtz-data-federation.de": [
"<local identifier>"
]
}
}
feudal_backend_host: "<hostname>"
debug:
# debug output from feudalScripts (aka feudalAdapters)
scripts: false
# debug communication with the backend
backend: true
# execute deployments in sequence and not parallel (-> nicer log output)
sequential_execution: false
username: '<username registerred at the portal>'
password: '<password registerred at the portal>'
services:
'<local identifier>':
name: '<service name>'
description: 'This is the foo service'
command: './fooService'
group_to_service_ids:
'<group name>':
- '<local identifier>'
entitlement_to_service_ids:
'<entitlement string>':
- '<local identifier>'
module git.scc.kit.edu/feudal/feudalClient
require (
git.scc.kit.edu/feudal/feudalScripts v1.1.0
git.scc.kit.edu/feudal/feudalScripts/v2 v2.0.0
github.com/alecthomas/jsonschema v0.0.0-20190122210438-a6952de1bbe6
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc // indirect
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/streadway/amqp v0.0.0-20181205114330-a314942b2fd9
github.com/stretchr/testify v1.3.0 // indirect
......@@ -11,4 +12,7 @@ require (
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xeipuuv/gojsonschema v1.1.0
gopkg.in/alecthomas/kingpin.v2 v2.2.6
gopkg.in/yaml.v2 v2.2.5
)
go 1.13
git.scc.kit.edu/feudal/feudalClient v0.1.0/go.mod h1:0h97fa5a+3UqI6rGnkcUhM4DyFvBt4zAXIja+EJbXls=
git.scc.kit.edu/feudal/feudalScripts v1.0.0/go.mod h1:5+SmQtrtjjLAI5rgZXVVRNvLfwlKw0uRDwWkFq8HUJo=
git.scc.kit.edu/feudal/feudalScripts v1.1.0 h1:/j1FUlvwT6gOyjIS7+NnpDEcRE/QZfxA0JX7TIfFNc8=
git.scc.kit.edu/feudal/feudalScripts v1.1.0/go.mod h1:IJmsus8xLMsKeb+n8Gqg2J4s6fRtiNwxXNyD+sC9OJU=
git.scc.kit.edu/feudal/feudalScripts/v2 v2.0.0 h1:3907HJ9XEZSuEZ0oAuijssG9SGFjAdmyDt8P7SlFFZY=
git.scc.kit.edu/feudal/feudalScripts/v2 v2.0.0/go.mod h1:EVXrOHOlnw3VZ02LZQDSN7bxldpJ7p31tfKZ5zqAEXA=
github.com/alecthomas/jsonschema v0.0.0-20190122210438-a6952de1bbe6 h1:xadBCbc8D9mmkaNfCsEBHbIoCjbayJXJNsY1JjPjNio=
......@@ -16,7 +12,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/streadway/amqp v0.0.0-20181107104731-27835f1a64e9/go.mod h1:1WNBiOZtZQLpVAyu0iTduoJL9hEsMloAK5XWrtW0xdY=
github.com/streadway/amqp v0.0.0-20181205114330-a314942b2fd9 h1:37QTz/gdHBLQcsmgMTnQDSWCtKzJ7YnfI2M2yTdr4BQ=
github.com/streadway/amqp v0.0.0-20181205114330-a314942b2fd9/go.mod h1:1WNBiOZtZQLpVAyu0iTduoJL9hEsMloAK5XWrtW0xdY=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
......@@ -30,3 +25,7 @@ github.com/xeipuuv/gojsonschema v1.1.0 h1:ngVtJC9TY/lg0AA/1k48FYhBrhRoFlEmWzsehp
github.com/xeipuuv/gojsonschema v1.1.0/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c=
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
package main
import (
"fmt"
"io/ioutil"
"log"
"net/http"
"os"
"path"
"path/filepath"
"git.scc.kit.edu/feudal/feudalClient/config"
deps "git.scc.kit.edu/feudal/feudalClient/deployments"
......@@ -23,7 +26,7 @@ var (
).Author(
"Lukas Burgey",
).Version(
"v2.1.2",
"v2.3.0",
)
cmdStart = app.Command("start", "Starts the client in its normal operation mode.").Default()
cmdDeregister = app.Command("deregister", "Before disabling the client: Use deregister to inform the backend that the client ceases operation.")
......@@ -49,7 +52,7 @@ func deregister(conf *config.Config) {
req, err = http.NewRequest(
"PUT",
"https://"+conf.Hostname+"/backend/clientapi/deregister",
"https://"+conf.Hostname+"/client/deregister",
nil,
)
if err != nil {
......@@ -114,8 +117,8 @@ func prepareDebugging(conf *config.Config) {
// Override the debug configuration with parameters
if *debugAll {
*scriptDebugging = true
*backendDebugging = true
conf.Debug.Backend = true
conf.Debug.Scripts = true
}
if *scriptDebugging {
conf.Debug.Scripts = true
......@@ -138,6 +141,80 @@ func prepareDebugging(conf *config.Config) {
}
}
// search for a config file
// dir pattern: $HOME/.config -> /etc/feudal -> $CWD
// file pattern: *.yaml -> *.json
func findConfigFile(arg string) (handle *os.File, err error) {
globString := func(gs string) string {
var files []string
files, err = filepath.Glob(gs)
if err == nil && len(files) > 0 {
log.Printf("Globbed: %v", files)
_, err = os.Stat(files[0])
if err == nil {
return files[0]
}
log.Printf("Error opening: %s", err)
}
return ""
}
globDir := func(ds string) (name string) {
name = globString(path.Join(ds, "*.yaml"))
if name != "" {
return
}
name = globString(path.Join(ds, "*.json"))
if name != "" {
return
}
return
}
findName := func() string {
name := ""
if *configFile != "" {
// check if file exists
_, err = os.Stat(*configFile)
if err == nil {
return *configFile
}
log.Printf("Error opening: %s", err)
}
var home, cwd string
// search $HOME/.config
if home, err = os.UserHomeDir(); err == nil {
name = globDir(path.Join(home, ".config/feudal"))
if name != "" {
return name
}
}
name = globDir("/etc/feudal")
if name != "" {
return name
}
cwd, err = os.Getwd()
if err == nil {
name = globDir(cwd)
if name != "" {
return name
}
}
return ""
}
firstName := findName()
if firstName == "" {
err = fmt.Errorf(`No config file!
Provide -c/--config or place a config json/yaml in $HOME/.config/feudal or /etc/feudal or the current working directory`)
return
}
return os.Open(firstName)
}
func main() {
var err error
......@@ -158,8 +235,15 @@ func main() {
return
}
// search appropriate config
handle, err := findConfigFile(*configFile)
if err != nil {
log.Fatalf("[Conf] Error opening config file: %s", err)
}
log.Printf("[Conf] Opened config file %s", handle.Name())
// read the config file
conf, err := config.ReadConfig(*configFile)
conf, err := config.ReadConfig(handle, handle.Name())
if err != nil {
log.Fatalf("[Conf] %s", err)
}
......
......@@ -9,7 +9,7 @@ import (
"os"
"git.scc.kit.edu/feudal/feudalClient/config"
scripts "git.scc.kit.edu/feudal/feudalScripts"
"git.scc.kit.edu/feudal/feudalScripts/v2"
"github.com/alecthomas/jsonschema"
"github.com/xeipuuv/gojsonschema"
)
......
......@@ -13,8 +13,8 @@ import (
"time"
"git.scc.kit.edu/feudal/feudalClient/config"
"git.scc.kit.edu/feudal/feudalClient/deployments"
"git.scc.kit.edu/feudal/feudalScripts/v2"
deps "git.scc.kit.edu/feudal/feudalClient/deployments"
scripts "git.scc.kit.edu/feudal/feudalScripts/v2"
)
type (
......@@ -236,7 +236,7 @@ func (sink *Sink) sendReply(reply deps.Reply) (err error) {
log.Printf("Dep Response:\n%s", taskResponse)
}
url := fmt.Sprintf("https://%s/backend/clientapi/dep-state", sink.Config.Hostname)
url := fmt.Sprintf("https://%s/client/dep-state", sink.Config.Hostname)
var req *http.Request
req, err = http.NewRequest("PATCH", url, bytes.NewReader(taskResponse))
if err != nil {
......
......@@ -10,7 +10,7 @@ import (
"time"
"git.scc.kit.edu/feudal/feudalClient/config"
"git.scc.kit.edu/feudal/feudalClient/deployments"
deps "git.scc.kit.edu/feudal/feudalClient/deployments"
)
type (
......@@ -35,7 +35,7 @@ func (src Source) fetchDeps(sink chan<- deps.Dep) (err error) {
}
// construct a request
uri := "https://" + src.Config.Hostname + "/backend/clientapi/dep-states"
uri := "https://" + src.Config.Hostname + "/client/dep-states"
req, err := http.NewRequest("GET", uri, nil)
if err != nil {
err = fmt.Errorf("Request creation failed: %s", err)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment