commit 4d7c9836008f88dcd15ebc800f82c81f5474b8e2 Author: Alex Vanin Date: Sat May 28 20:48:56 2022 +0300 Initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c5e82d7 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +bin \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..e27d0d9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 Alexey Vanin + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..b09bf85 --- /dev/null +++ b/Makefile @@ -0,0 +1,7 @@ +VERSION ?= $(shell git describe --tags --always 2>/dev/null) + +build: + go build -ldflags "-X main.Version=$(VERSION)" -o ./bin/nezabx + +clean: + rm -rf ./bin \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..7fbf825 --- /dev/null +++ b/README.md @@ -0,0 +1,35 @@ +# Nezabx + +## Overview + +Nezabx (njɛ-za-bɪks) - simple periodic job scheduler with notifications. + +Supported notification channels: +- email + +Supported jobs: +- arbitrary shell commands + +## Build + +``` +$ make +go build -ldflags "-X main.Version=v0.1.0" -o ./bin/nezabx +``` + +## Config + +See configuration example with comments in [example.yaml](/example.yaml) + +## Run + +``` +$ ./bin/nezabx -c config.yaml +2022-05-28T22:47:10.979+0300 info application started +2022-05-28T22:48:00.049+0300 info script run ok {"cmd": "./healthcheck.sh arg", "next iteration at": "2022-05-28T22:49:00.000+0300"} +^C2022-05-28T22:48:05.354+0300 info application received termination signal +``` + +## License + +Source code is available under the [MIT License](/LICENSE). \ No newline at end of file diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..a727030 --- /dev/null +++ b/TODO.md @@ -0,0 +1,5 @@ +- [ ] Check config relations between runner and notification groups on start +- [ ] Add option to send "incident resolved" notifications +- [ ] Add Matrix notificator +- [ ] Add Telegram notificator +- [ ] Unit tests for runners, mock interfaces \ No newline at end of file diff --git a/config.go b/config.go new file mode 100644 index 0000000..041bb77 --- /dev/null +++ b/config.go @@ -0,0 +1,68 @@ +package main + +import ( + "fmt" + "os" + "time" + + "gopkg.in/yaml.v3" +) + +type ( + Config struct { + State StateConfig `yaml:"state"` + Notifications NotificationsConfig `yaml:"notifications"` + Commands []CommandConfig `yaml:"commands"` + } + + StateConfig struct { + Bolt string `yaml:"bolt"` + } + + NotificationsConfig struct { + Email *EmailConfig `yaml:"email"` + } + + EmailConfig struct { + SMTP string `yaml:"smtp"` + Login string `yaml:"login"` + Password string `yaml:"password"` + Groups []EmailGroupConfig `yaml:"groups"` + } + + EmailGroupConfig struct { + Name string `yaml:"name"` + Addresses []string `yaml:"addresses"` + } + + CommandConfig struct { + Name string `yaml:"name"` + Exec string `yaml:"exec"` + Threshold uint `yaml:"threshold"` + ThresholdSleep time.Duration `yaml:"threshold_sleep"` + Cron string `yaml:"cron"` + Interval time.Duration `yaml:"interval"` + Timeout time.Duration `yaml:"timeout"` + Notifications []string `yaml:"notifications"` + } +) + +var Version = "dev" + +func ReadConfig(file string) (*Config, error) { + data, err := os.ReadFile(file) + if err != nil { + return nil, fmt.Errorf("config: %w", err) + } + c := new(Config) + err = yaml.Unmarshal(data, c) + if err != nil { + return nil, fmt.Errorf("config: %w", err) + } + return c, validateConfig(c) +} + +func validateConfig(_ *Config) error { + // todo(alexvanin): set defaults and validate values such as bad email addresses + return nil +} diff --git a/db/bolt.go b/db/bolt.go new file mode 100644 index 0000000..27acdcb --- /dev/null +++ b/db/bolt.go @@ -0,0 +1,64 @@ +package db + +import ( + "encoding/json" + "fmt" + "path" + + "go.etcd.io/bbolt" +) + +type ( + Bolt struct { + db *bbolt.DB + } + + Status struct { + ID []byte + Failed bool + Notified bool + } +) + +var ( + statusBucket = []byte("status") +) + +func NewBolt(filename string) (*Bolt, error) { + dbPath := path.Join(filename) + + db, err := bbolt.Open(dbPath, 0600, nil) + if err != nil { + return nil, fmt.Errorf("bolot init: %w", err) + } + + return &Bolt{db}, nil +} + +func (b *Bolt) Status(id []byte) (st Status, err error) { + return st, b.db.Update(func(tx *bbolt.Tx) error { + bkt, err := tx.CreateBucketIfNotExists(statusBucket) + if err != nil { + return err + } + v := bkt.Get(id) + if len(v) == 0 { + return nil + } + return json.Unmarshal(v, &st) + }) +} + +func (b *Bolt) SetStatus(id []byte, st Status) error { + return b.db.Update(func(tx *bbolt.Tx) error { + bkt, err := tx.CreateBucketIfNotExists(statusBucket) + if err != nil { + return err + } + v, err := json.Marshal(st) + if err != nil { + return err + } + return bkt.Put(id, v) + }) +} diff --git a/example.yaml b/example.yaml new file mode 100644 index 0000000..378806a --- /dev/null +++ b/example.yaml @@ -0,0 +1,25 @@ +--- + +state: + bolt: ./nz.state # path to local state file + +notifications: + email: + smtp: smtp.gmail.com:587 # SMTP server address + login: nzbx@corp.com # email sender login + password: secret # email sender password + groups: # specify groups of alert receivers + - name: developers # group name + addresses: # list of alert receiver addresses in this group + - alex@corp.com + +commands: + - name: Test command # short job description; all jobs MUST have unique description + exec: ./script.sh arg1 # command, application or script to execute + cron: "*/5 * * * *" # schedule execution as cron table record + interval: 10s # overrides cron; schedule execution by interval between consecutive executions + timeout: 2s # time limit for successful execution + threshold: 3 # amount of consecutive failures before sending notifications + threshold_sleep: 5s # interval between two failed executions before reaching threshold + notifications: # specify 'type:group' tuple to send notifications + - email:developers diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..96556bc --- /dev/null +++ b/go.mod @@ -0,0 +1,17 @@ +module github.com/alexvanin/nezabx + +go 1.18 + +require ( + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 + github.com/robfig/cron/v3 v3.0.0 + go.etcd.io/bbolt v1.3.6 + go.uber.org/zap v1.21.0 + gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99 +) + +require ( + go.uber.org/atomic v1.9.0 // indirect + go.uber.org/multierr v1.8.0 // indirect + golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..1877ac4 --- /dev/null +++ b/go.sum @@ -0,0 +1,71 @@ +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/robfig/cron/v3 v3.0.0 h1:kQ6Cb7aHOHTSzNVNEhmp8EcWKLb4CbiMW9h9VyIhO4E= +github.com/robfig/cron/v3 v3.0.0/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= +go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= +go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= +go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= +go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99 h1:dbuHpmKjkDzSOMKAWl10QNlgaZUd3V1q99xc81tt2Kc= +gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go new file mode 100644 index 0000000..40fc2d6 --- /dev/null +++ b/main.go @@ -0,0 +1,107 @@ +package main + +import ( + "context" + "flag" + "fmt" + "os" + "os/signal" + + "github.com/alexvanin/nezabx/db" + "github.com/alexvanin/nezabx/notifications/email" + "github.com/alexvanin/nezabx/runners" + "github.com/robfig/cron/v3" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +func main() { + configFile := flag.String("c", "", "config file") + debugFlag := flag.Bool("debug", false, "debug mode") + versionFlag := flag.Bool("version", false, "show version") + flag.Parse() + + if *versionFlag { + fmt.Printf("Nezabx %s\n", Version) + os.Exit(0) + } + + ctx, _ := signal.NotifyContext(context.Background(), os.Interrupt) + log := Logger(*debugFlag) + + cfg, err := ReadConfig(*configFile) + if err != nil { + log.Error("invalid configuration", zap.Error(err)) + os.Exit(1) + } + + state, err := db.NewBolt(cfg.State.Bolt) + if err != nil { + log.Error("invalid configuration", zap.Error(err)) + os.Exit(1) + } + + var mailNotificator *email.Notificator + + if cfg.Notifications.Email != nil { + mailNotificator := email.NewNotificator( + cfg.Notifications.Email.SMTP, + cfg.Notifications.Email.Login, + cfg.Notifications.Email.Password, + ) + for _, group := range cfg.Notifications.Email.Groups { + mailNotificator.AddGroup(group.Name, group.Addresses) + } + } + + for _, command := range cfg.Commands { + cmd, err := runners.NewCommand(command.Exec) + if err != nil { + log.Warn("invalid command configuration", zap.String("command", command.Name), zap.Error(err)) + os.Exit(1) + } + var cronSchedule cron.Schedule + if command.Interval == 0 { + cronSchedule, err = cron.ParseStandard(command.Cron) + if err != nil { + log.Warn("invalid cron configuration", zap.String("command", command.Name), zap.Error(err)) + os.Exit(1) + } + } + commandRunner := runners.CommandRunner{ + Log: log, + DB: state, + MailNotificator: mailNotificator, + Command: cmd, + Name: command.Name, + Threshold: command.Threshold, + ThresholdSleep: command.ThresholdSleep, + Timeout: command.Timeout, + Interval: command.Interval, + CronSchedule: cronSchedule, + Notifications: command.Notifications, + } + commandRunner.Run(ctx) + } + + log.Info("application started") + + <-ctx.Done() + + log.Info("application received termination signal") +} + +func Logger(debug bool) *zap.Logger { + logCfg := zap.NewProductionConfig() + logCfg.Level.SetLevel(zap.InfoLevel) + logCfg.DisableCaller = true + logCfg.Encoding = "console" + logCfg.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder + logCfg.DisableStacktrace = true + if debug { + logCfg.Level.SetLevel(zap.DebugLevel) + logCfg.DisableCaller = false + } + logger, _ := logCfg.Build() + return logger +} diff --git a/notifications/email/email.go b/notifications/email/email.go new file mode 100644 index 0000000..7c38ec1 --- /dev/null +++ b/notifications/email/email.go @@ -0,0 +1,71 @@ +package email + +import ( + "errors" + "fmt" + "net/smtp" + "strings" +) + +type ( + Sender struct { + SMTPServer string + Login string + Password string + } + + Notificator struct { + Sender Sender + Groups map[string][]string + } +) + +func NewNotificator(smtp, login, password string) *Notificator { + return &Notificator{ + Sender: Sender{ + SMTPServer: smtp, + Login: login, + Password: password, + }, + Groups: make(map[string][]string), + } +} + +func (n *Notificator) AddGroup(name string, addresses []string) { + n.Groups[name] = addresses +} + +func (n Notificator) Send(group, sub, body string) error { + receivers, ok := n.Groups[group] + if !ok { + return errors.New("unknown group") + } + return n.Sender.SendMail(receivers, sub, body) +} + +func (s Sender) SendMail(to []string, subj, body string) error { + toTxt := fmt.Sprintf("To: %s\r\n", strings.Join(to, ";")) + subjTxt := fmt.Sprintf("Subject: %s\r\n", subj) + mimeTxt := "MIME-version: 1.0;\r\nContent-Type: text/html; charset=\"UTF-8\";\r\n\r\n" + bodyTxt := fmt.Sprintf("%s\r\n", body) + mailTxt := toTxt + subjTxt + mimeTxt + bodyTxt + return smtp.SendMail(s.SMTPServer, s, s.Login, to, []byte(mailTxt)) +} + +func (s Sender) Start(_ *smtp.ServerInfo) (string, []byte, error) { + return "LOGIN", []byte{}, nil +} + +func (s Sender) Next(fromServer []byte, more bool) ([]byte, error) { + if more { + switch str := string(fromServer); str { + case "Username:": + return []byte(s.Login), nil + case "Password:": + return []byte(s.Password), nil + default: + return nil, fmt.Errorf("unknown fromServer %s", str) + } + } + return nil, nil +} diff --git a/runners/command.go b/runners/command.go new file mode 100644 index 0000000..26c846b --- /dev/null +++ b/runners/command.go @@ -0,0 +1,231 @@ +package runners + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "os/exec" + "strings" + "time" + + "github.com/alexvanin/nezabx/db" + "github.com/alexvanin/nezabx/notifications/email" + "github.com/google/shlex" + "github.com/robfig/cron/v3" + "go.uber.org/zap" +) + +type ( + Command struct { + orig string + cmd string + args []string + } + + CommandRunner struct { + Log *zap.Logger + DB *db.Bolt + MailNotificator *email.Notificator + Command *Command + Name string + Threshold uint + ThresholdSleep time.Duration + Timeout time.Duration + Interval time.Duration + CronSchedule cron.Schedule + Notifications []string + } +) + +func NewCommand(path string) (*Command, error) { + command, err := shlex.Split(path) + if err != nil { + return nil, err + } + if len(command) < 1 { + return nil, errors.New("empty command") + } + return &Command{ + orig: path, + cmd: command[0], + args: command[1:], + }, nil +} + +func (h Command) Exec() ([]byte, error) { + cmd := exec.Command(h.cmd, h.args...) + return cmd.CombinedOutput() +} + +func (h Command) String() string { + return h.orig +} + +func (c CommandRunner) Run(ctx context.Context) { + h := sha256.Sum256([]byte(c.Name)) + id := h[:] + + go func() { + firstIter := true + for { + if !firstIter || c.CronSchedule == nil { + c.run(ctx, id) + } + firstIter = false + select { + case <-ctx.Done(): + return + case <-time.After(c.untilNextIteration()): + } + } + }() +} + +func (c CommandRunner) run(ctx context.Context, id []byte) { + st, err := c.DB.Status(id) + if err != nil { + c.Log.Warn("database is broken", zap.String("id", hex.EncodeToString(id)), zap.Error(err)) + return + } + + c.Log.Debug("starting script", zap.Stringer("cmd", c.Command)) + output, err := execScript(ctx, c.Command, c.Timeout) + if err == nil { + c.processSuccessfulExecution(id, st) + return + } + + if !st.Notified { + for i := 1; i < int(c.Threshold); i++ { + c.Log.Info("script run failed", zap.Stringer("cmd", c.Command), zap.Int("iteration", i)) + select { + case <-ctx.Done(): + return + case <-time.After(c.ThresholdSleep): + } + + output, err = execScript(ctx, c.Command, c.Timeout) + if err == nil { + c.processSuccessfulExecution(id, st) + return + } + } + } + + c.processFailedExecution(id, st, output, err) +} + +func (c CommandRunner) processSuccessfulExecution(id []byte, st db.Status) { + if !st.Failed { + c.Log.Info("script run ok", zap.Stringer("cmd", c.Command), + zap.Time("next iteration at", c.nextIteration())) + return + } + + c.Log.Info("script run ok, recovered after failure", zap.Stringer("cmd", c.Command), + zap.Time("next iteration at", c.nextIteration())) + + st.Failed = false + st.Notified = false + err := c.DB.SetStatus(id, st) + if err != nil { + c.Log.Warn("database is broken", zap.String("id", hex.EncodeToString(id)), zap.Error(err)) + } +} + +func (c CommandRunner) processFailedExecution(id []byte, st db.Status, output []byte, err error) { + if st.Failed { + c.Log.Info("script run failed, notification has already been sent", + zap.Stringer("cmd", c.Command), + zap.Time("next iteration at", c.nextIteration())) + return + } + + c.Log.Info("script run failed, sending notification", + zap.Stringer("cmd", c.Command), + zap.Time("next iteration at", c.nextIteration())) + st.Failed = true + st.Notified = true + err = c.notify(output, err) + if err != nil { + c.Log.Warn("notification was not sent", zap.Stringer("cmd", c.Command), zap.Error(err)) + st.Notified = false + } + + err = c.DB.SetStatus(id, st) + if err != nil { + c.Log.Warn("database is broken", zap.String("id", hex.EncodeToString(id)), zap.Error(err)) + } +} + +func (c CommandRunner) notify(out []byte, err error) error { + msg := fmt.Sprintf("Script runner \"%s\" has failed.
"+ + "Executed command: %s
"+ + "Exit error: %s

"+ + "Terminal output:
%s
", c.Name, c.Command, err.Error(), out) + + for _, target := range c.Notifications { + kv := strings.Split(target, ":") + if len(kv) != 2 { + c.Log.Warn("invalid notification target", zap.String("value", target)) + continue + } + switch kv[0] { + case "email": + if c.MailNotificator == nil { + c.Log.Warn("email notifications were not configured") + continue + } + err = c.MailNotificator.Send(kv[1], "Nezabx alert message", msg) + if err != nil { + return err + } + default: + c.Log.Warn("invalid notification type", zap.String("value", target)) + continue + } + } + return nil +} + +func (c CommandRunner) untilNextIteration() time.Duration { + if c.CronSchedule != nil { + return time.Until(c.CronSchedule.Next(time.Now())) + } + return c.Interval +} + +func (c CommandRunner) nextIteration() time.Time { + if c.CronSchedule != nil { + return c.CronSchedule.Next(time.Now()) // no truncate because cron operates at most with minute truncated values + } + return time.Now().Add(c.Interval).Truncate(time.Second) +} + +func execScript(ctx context.Context, cmd *Command, timeout time.Duration) (output []byte, err error) { + type cmdOutput struct { + out []byte + err error + } + + res := make(chan cmdOutput) + go func(ch chan<- cmdOutput) { + v, err := cmd.Exec() + res <- cmdOutput{ + out: v, + err: err, + } + close(res) + }(res) + + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(timeout): + return nil, errors.New("script execution timeout") + case v := <-res: + return v.out, v.err + } +}