diff --git a/README.md b/README.md index 68bf7f0..63780a7 100644 --- a/README.md +++ b/README.md @@ -306,6 +306,13 @@ Release: Use the `-sentry-release` flag or the `SENTRY_RELEASE` environment vari $ ./supercronic -sentry-dsn YOUR_SENTRY_DSN -sentry-release YOUR_RELEASE ``` +Extra-trace: Use the `-sentry-extra-trace` flag to include the stdout and stderr output of failed jobs in the Sentry event’s extra data. +This provides additional context in Sentry without altering normal logging behavior. + +``` +$ ./supercronic -sentry-dsn YOUR_SENTRY_DSN -sentry-extra-trace true +``` + ## Questions and Support ### Please feel free to open an issue in this repository if you have any question diff --git a/cron/cron.go b/cron/cron.go index 512dc5c..1b7f97b 100644 --- a/cron/cron.go +++ b/cron/cron.go @@ -3,6 +3,7 @@ package cron import ( "bufio" "context" + "errors" "fmt" "io" "os" @@ -20,9 +21,51 @@ import ( var ( READ_BUFFER_SIZE = 64 * 1024 + TAIL_BUFFER_SIZE = 64 * 1024 ) -func startReaderDrain(wg *sync.WaitGroup, readerLogger *logrus.Entry, reader io.ReadCloser) { +// ringBuffer garde les N derniers octets écrits (tail). +type ringBuffer struct { + mu sync.Mutex + buf []byte + max int +} + +func newRingBuffer(max int) *ringBuffer { + return &ringBuffer{max: max} +} + +func (r *ringBuffer) Write(p []byte) (int, error) { + r.mu.Lock() + defer r.mu.Unlock() + if r.max <= 0 { + return len(p), nil + } + if len(r.buf)+len(p) <= r.max { + r.buf = append(r.buf, p...) + return len(p), nil + } + needDrop := len(r.buf) + len(p) - r.max + if needDrop >= len(r.buf) { + if len(p) >= r.max { + r.buf = append([]byte{}, p[len(p)-r.max:]...) + } else { + r.buf = make([]byte, 0, r.max) + r.buf = append(r.buf, p...) + } + return len(p), nil + } + r.buf = append(r.buf[needDrop:], p...) + return len(p), nil +} + +func (r *ringBuffer) String() string { + r.mu.Lock() + defer r.mu.Unlock() + return string(r.buf) +} + +func startReaderDrain(wg *sync.WaitGroup, readerLogger *logrus.Entry, reader io.ReadCloser, tail *ringBuffer) { wg.Add(1) go func() { @@ -55,6 +98,9 @@ func startReaderDrain(wg *sync.WaitGroup, readerLogger *logrus.Entry, reader io. } readerLogger.Info(string(line)) + if tail != nil { + _, _ = tail.Write(append(line, '\n')) + } if isPrefix { readerLogger.Warn("last line exceeded buffer size, continuing...") @@ -63,7 +109,31 @@ func startReaderDrain(wg *sync.WaitGroup, readerLogger *logrus.Entry, reader io. }() } -func runJob(cronCtx *crontab.Context, command string, jobLogger *logrus.Entry, passthroughLogs bool) error { +func handleExecError(err error) (exitCode int, wrappedErr error) { + if err == nil { + return 0, nil + } + + exitCode = 1 + wrappedErr = fmt.Errorf("error running command: %v", err) + + var ee *exec.ExitError + if errors.As(err, &ee) { + if ws, ok := ee.Sys().(syscall.WaitStatus); ok { + exitCode = ws.ExitStatus() + } + } + + return exitCode, wrappedErr +} + +func runJob( + cronCtx *crontab.Context, + command string, + jobLogger *logrus.Entry, + passthroughLogs bool, +) (err error, stdoutTailStr string, stderrTailStr string, exitCode int, duration time.Duration) { + jobLogger.Info("starting") cmd := exec.Command(cronCtx.Shell, "-c", command) @@ -80,46 +150,62 @@ func runJob(cronCtx *crontab.Context, command string, jobLogger *logrus.Entry, p var stdout io.ReadCloser = nil var stderr io.ReadCloser = nil - var err error + exitCode = 0 + + var stdoutTail, stderrTail *ringBuffer if passthroughLogs { cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr } else { - stdout, err = cmd.StdoutPipe() - if err != nil { - return err + var e error + stdout, e = cmd.StdoutPipe() + if e != nil { + err = e + return } - - stderr, err = cmd.StderrPipe() - if err != nil { - return err + stderr, e = cmd.StderrPipe() + if e != nil { + err = e + return } + stdoutTail = newRingBuffer(TAIL_BUFFER_SIZE) + stderrTail = newRingBuffer(TAIL_BUFFER_SIZE) } - if err := cmd.Start(); err != nil { - return err + start := time.Now() + startErr := cmd.Start() + exitCode, err = handleExecError(startErr) + if exitCode != 0 { + return } var wg sync.WaitGroup if stdout != nil { stdoutLogger := jobLogger.WithFields(logrus.Fields{"channel": "stdout"}) - startReaderDrain(&wg, stdoutLogger, stdout) + startReaderDrain(&wg, stdoutLogger, stdout, stdoutTail) } if stderr != nil { stderrLogger := jobLogger.WithFields(logrus.Fields{"channel": "stderr"}) - startReaderDrain(&wg, stderrLogger, stderr) + startReaderDrain(&wg, stderrLogger, stderr, stderrTail) } wg.Wait() - if err := cmd.Wait(); err != nil { - return fmt.Errorf("error running command: %v", err) + waitErr := cmd.Wait() + duration = time.Since(start) + + if stdoutTail != nil { + stdoutTailStr = stdoutTail.String() + } + if stderrTail != nil { + stderrTailStr = stderrTail.String() } - return nil + exitCode, err = handleExecError(waitErr) + return } func monitorJob(ctx context.Context, job *crontab.Job, t0 time.Time, jobLogger *logrus.Entry, overlapping bool, promMetrics *prometheus_metrics.PrometheusMetrics) { @@ -222,6 +308,7 @@ func StartJob( cronLogger *logrus.Entry, overlapping bool, passthroughLogs bool, + sentryExtraTrace bool, promMetrics *prometheus_metrics.PrometheusMetrics, ) { runThisJob := func(t0 time.Time, jobLogger *logrus.Entry) { @@ -242,7 +329,7 @@ func StartJob( defer timer.ObserveDuration() - err := runJob(cronCtx, job.Command, jobLogger, passthroughLogs) + err, stdout, stderr, exitCode, duration := runJob(cronCtx, job.Command, jobLogger, passthroughLogs) promMetrics.CronsExecCounter.With(jobPromLabels(job)).Inc() @@ -251,7 +338,16 @@ func StartJob( promMetrics.CronsSuccessCounter.With(jobPromLabels(job)).Inc() } else { - jobLogger.Error(err) + fields := logrus.Fields{ + "job.exit_code": exitCode, + "job.duration": duration, + } + if sentryExtraTrace == true { + fields["job.stdout"] = stdout + fields["job.stderr"] = stderr + } + + jobLogger.WithFields(fields).Error(err) promMetrics.CronsFailCounter.With(jobPromLabels(job)).Inc() } diff --git a/cron/cron_test.go b/cron/cron_test.go index 966a31b..2f767ff 100644 --- a/cron/cron_test.go +++ b/cron/cron_test.go @@ -150,7 +150,7 @@ func TestRunJob(t *testing.T) { label := fmt.Sprintf("RunJob(%q)", tt.command) logger, channel := newTestLogger() - err := runJob(tt.context, tt.command, logger, false) + err, _, _, _, _ := runJob(tt.context, tt.command, logger, false) if tt.success { assert.Nil(t, err, label) } else { @@ -198,7 +198,7 @@ func TestStartJobExitsOnRequest(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - StartJob(&wg, &basicContext, &job, ctx, logger, false, false, &PROM_METRICS) + StartJob(&wg, &basicContext, &job, ctx, logger, false, false, false, &PROM_METRICS) wg.Wait() } @@ -218,7 +218,7 @@ func TestStartJobRunsJob(t *testing.T) { logger, channel := newTestLogger() - StartJob(&wg, &basicContext, &job, ctx, logger, false, false, &PROM_METRICS) + StartJob(&wg, &basicContext, &job, ctx, logger, false, false, false, &PROM_METRICS) select { case entry := <-channel: diff --git a/cronexpr/cronexpr/go.mod b/cronexpr/cronexpr/go.mod index fa9e108..6401353 100644 --- a/cronexpr/cronexpr/go.mod +++ b/cronexpr/cronexpr/go.mod @@ -1,8 +1,6 @@ module github.com/aptible/supercronic/cronexpr/cronexpr -go 1.24.6 - -toolchain go1.24.7 +go 1.25.2 replace github.com/aptible/supercronic => ../../ diff --git a/go.mod b/go.mod index 4b4a64c..6c66949 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.67.1 // indirect github.com/prometheus/procfs v0.17.0 // indirect - github.com/rogpeppe/go-internal v1.12.0 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect golang.org/x/sys v0.37.0 // indirect google.golang.org/protobuf v1.36.10 // indirect diff --git a/go.sum b/go.sum index 34a9c2e..a9f1231 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,14 @@ +github.com/alecthomas/kingpin/v2 v2.4.0 h1:f48lwail6p8zpO1bC4TxtqACaGqHYA22qkHjHpqDjYY= +github.com/alecthomas/kingpin/v2 v2.4.0/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= +github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b h1:mimo19zliBX/vSQ6PWWSL9lK8qwHozUj03+zLoEB8O0= +github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b/go.mod h1:fvzegU4vN3H1qMT+8wDmzjAcDONcgo2/SZ/TyfdUOFs= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d h1:S2NE3iHSwP0XV47EEXL8mWmRdEfGscSJ+7EgePNgt0s= github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -14,8 +19,16 @@ github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs= github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= +github.com/golang/protobuf v1.5.0 h1:LUVKkCeviFUMKqHa4tXIIij/lbhnMbP7Fn5wKdKkRh4= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -24,8 +37,14 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -40,19 +59,36 @@ github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7D github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc= +github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= +golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I= +golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY= +golang.org/x/oauth2 v0.31.0 h1:8Fq0yVZLh4j4YA47vHKFTa9Ew5XIrCP8LC6UeNZnLxo= +golang.org/x/oauth2 v0.31.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= +golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= +golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= +golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/main.go b/main.go index 8c6de95..8580614 100644 --- a/main.go +++ b/main.go @@ -51,6 +51,7 @@ func main() { sentry := flag.String("sentry-dsn", "", "enable Sentry error logging, using provided DSN") sentryEnvironmentFlag := flag.String("sentry-environment", "", "specify the application's environment for Sentry error reporting") sentryReleaseFlag := flag.String("sentry-release", "", "specify the application's release version for Sentry error reporting") + sentryExtraTrace := flag.Bool("sentry-extra-trace", false, "log extra trace information to Sentry (e.g. stdout/stderr) default: false") sentryAlias := flag.String("sentryDsn", "", "alias for sentry-dsn") overlapping := flag.Bool("overlapping", false, "enable tasks overlapping") flag.Parse() @@ -254,7 +255,7 @@ func main() { "job.position": job.Position, }) - cron.StartJob(&wg, tab.Context, job, exitCtx, cronLogger, *overlapping, *passthroughLogs, &promMetrics) + cron.StartJob(&wg, tab.Context, job, exitCtx, cronLogger, *overlapping, *passthroughLogs, *sentryExtraTrace, &promMetrics) } termSig := <-termChan