Hello, Beats friends,
We encountered an issue when we using Heartbeat in our environment.
After some further digging, we believe it cased by using a third-party module -- https://github.com/gorhill/cronexpr without considering the resource competing.
And since this third-party library is already archived, I'd like to file a BUG here.
The stack of panic are as following:
panic: runtime error: index out of range [30] with length 30
goroutine 502007 [running]:
github.com/gorhill/cronexpr.(*Expression).nextDayOfMonth(0xc004efba40, 0xbfd564d7c00940c7, 0x6b73094b488, 0x3ef3500, 0x18, 0xeb557a, 0x3c)
/go/beats-ebay/vendor/github.com/gorhill/cronexpr/cronexpr_next.go:114 +0x279
github.com/gorhill/cronexpr.(*Expression).nextHour(0xc004efba40, 0xbfd564d7c00940c7, 0x6b73094b488, 0x3ef3500, 0x3c, 0xc000992d70, 0xeb557a)
/go/beats-ebay/vendor/github.com/gorhill/cronexpr/cronexpr_next.go:129 +0x27f
github.com/gorhill/cronexpr.(*Expression).nextMinute(0xc004efba40, 0xbfd564d7c00940c7, 0x6b73094b488, 0x3ef3500, 0x1, 0x0, 0xc000992e00)
/go/beats-ebay/vendor/github.com/gorhill/cronexpr/cronexpr_next.go:150 +0x2e5
github.com/gorhill/cronexpr.(*Expression).nextSecond(0xc004efba40, 0xbfd564d7c00940c7, 0x6b73094b488, 0x3ef3500, 0x0, 0x9, 0x1e)
/go/beats-ebay/vendor/github.com/gorhill/cronexpr/cronexpr_next.go:174 +0x33c
github.com/gorhill/cronexpr.(*Expression).Next(0xc004efba40, 0xbfd564d7c00940c7, 0x6b73094b488, 0x3ef3500, 0xc0053d6e70, 0x2d46940, 0xc0037df800)
/go/beats-ebay/vendor/github.com/gorhill/cronexpr/cronexpr.go:234 +0x5dd
github.com/elastic/beats/v7/heartbeat/scheduler/schedule/cron.(*Schedule).Next(0xc004efba40, 0xbfd564d7c00940c7, 0x6b73094b488, 0x3ef3500, 0x6b73094b488, 0xc0053d6e70, 0x3ef3500)
/go/beats-ebay/vendor/github.com/elastic/beats/v7/heartbeat/scheduler/schedule/cron/cron.go:43 +0x49
github.com/elastic/beats/v7/heartbeat/scheduler.(*Scheduler).Add.func1(0xbfd564d7c0020fa9, 0x6b7308d836f, 0x3ef3500)
/go/beats-ebay/vendor/github.com/elastic/beats/v7/heartbeat/scheduler/scheduler.go:194 +0x112
created by github.com/elastic/beats/v7/heartbeat/scheduler.(*Scheduler).runOnce.func1
and this one:
panic: runtime error: index out of range [31] with length 30
goroutine 4105 [running]:
github.com/gorhill/cronexpr.(*Expression).nextDayOfMonth(0xc0001fefc0, 0xbfcb819680122f55, 0x4b8ff4f1c, 0x3eec100, 0x18, 0xeb457a, 0x3c)
/go/beats-ebay/vendor/github.com/gorhill/cronexpr/cronexpr_next.go:114 +0x279
github.com/gorhill/cronexpr.(*Expression).nextHour(0xc0001fefc0, 0xbfcb819680122f55, 0x4b8ff4f1c, 0x3eec100, 0x3c, 0xc0014b7568, 0xeb457a)
/go/beats-ebay/vendor/github.com/gorhill/cronexpr/cronexpr_next.go:129 +0x27f
github.com/gorhill/cronexpr.(*Expression).nextMinute(0xc0001fefc0, 0xbfcb819680122f55, 0x4b8ff4f1c, 0x3eec100, 0x1, 0x0, 0xc0014b75f8)
/go/beats-ebay/vendor/github.com/gorhill/cronexpr/cronexpr_next.go:150 +0x2e5
github.com/gorhill/cronexpr.(*Expression).nextSecond(0xc0001fefc0, 0xbfcb819680122f55, 0x4b8ff4f1c, 0x3eec100, 0x0, 0x8, 0x1f)
/go/beats-ebay/vendor/github.com/gorhill/cronexpr/cronexpr_next.go:174 +0x33c
github.com/gorhill/cronexpr.(*Expression).Next(0xc0001fefc0, 0xbfcb819680122f55, 0x4b8ff4f1c, 0x3eec100, 0xc002bec730, 0x10, 0x10)
/go/beats-ebay/vendor/github.com/gorhill/cronexpr/cronexpr.go:234 +0x5dd
github.com/elastic/beats/v7/heartbeat/scheduler/schedule/cron.(*Schedule).Next(0xc0001fefc0, 0xbfcb819680122f55, 0x4b8ff4f1c, 0x3eec100, 0x0, 0x3d078a0, 0xc001d00000)
/go/beats-ebay/vendor/github.com/elastic/beats/v7/heartbeat/scheduler/schedule/cron/cron.go:43 +0x49
github.com/elastic/beats/v7/heartbeat/monitors/wrappers.timespan(0xbfcb819680122f55, 0x4b8ff4f1c, 0x3eec100, 0xc0018a6fb0, 0x2540be400, 0xc002bbec00)
/go/beats-ebay/vendor/github.com/elastic/beats/v7/heartbeat/monitors/wrappers/monitors.go:94 +0x6f
github.com/elastic/beats/v7/heartbeat/monitors/wrappers.addMonitorMeta.func1.1(0xc001f22000, 0xc001f24000, 0x0, 0x0, 0x0, 0x0)
/go/beats-ebay/vendor/github.com/elastic/beats/v7/heartbeat/monitors/wrappers/monitors.go:76 +0x338
github.com/elastic/beats/v7/heartbeat/monitors/jobs.Wrap.func1(0xc001f22000, 0xe1354d, 0x1e, 0x7ff897fb8200, 0x20300000000000, 0x1980befff)
/go/beats-ebay/vendor/github.com/elastic/beats/v7/heartbeat/monitors/jobs/job.go:72 +0x4a
github.com/elastic/beats/v7/heartbeat/monitors/wrappers.makeAddSummary.func2.1(0xc001f22000, 0x0, 0x0, 0x0, 0x0, 0x0)
/go/beats-ebay/vendor/github.com/elastic/beats/v7/heartbeat/monitors/wrappers/monitors.go:176 +0x8e
github.com/elastic/beats/v7/heartbeat/monitors/jobs.Wrap.func1(0xc001f22000, 0xc001f22000, 0x0, 0x203000, 0x400, 0x7ff86c67c858)
/go/beats-ebay/vendor/github.com/elastic/beats/v7/heartbeat/monitors/jobs/job.go:72 +0x4a
github.com/elastic/beats/v7/heartbeat/monitors.runPublishJob(0xc0019658c0, 0x2d2c3e0, 0xc0019db830, 0x0, 0xc0019df000, 0x27f49c62280000)
/go/beats-ebay/vendor/github.com/elastic/beats/v7/heartbeat/monitors/task.go:140 +0x7c
github.com/elastic/beats/v7/heartbeat/monitors.(*configuredJob).prepareSchedulerJob.func1(0x2d41ea0, 0xc0019df040, 0x3eec100, 0x1, 0x0)
/go/beats-ebay/vendor/github.com/elastic/beats/v7/heartbeat/monitors/task.go:92 +0x46
github.com/elastic/beats/v7/heartbeat/scheduler.(*Scheduler).runRecursiveTask(0xc000268000, 0x2d41ea0, 0xc0019df040, 0xc0019eb5c0, 0xc001f1e000, 0xbfcb819680107da7, 0x4b8fd9d58, 0x3eec100)
/go/beats-ebay/vendor/github.com/elastic/beats/v7/heartbeat/scheduler/scheduler.go:270 +0x195
github.com/elastic/beats/v7/heartbeat/scheduler.(*Scheduler).runRecursiveJob(0xc000268000, 0x2d41ea0, 0xc0019df040, 0xc0019eb5c0, 0x0, 0x0, 0x0)
/go/beats-ebay/vendor/github.com/elastic/beats/v7/heartbeat/scheduler/scheduler.go:233 +0x89
github.com/elastic/beats/v7/heartbeat/scheduler.(*Scheduler).Add.func1(0xbfcb819680019472, 0x4b8eeb4d7, 0x3eec100)
/go/beats-ebay/vendor/github.com/elastic/beats/v7/heartbeat/scheduler/scheduler.go:192 +0xa2
created by github.com/elastic/beats/v7/heartbeat/scheduler.(*Scheduler).runOnce.func1
/go/beats-ebay/vendor/github.com/elastic/beats/v7/heartbeat/scheduler/scheduler.go:223 +0x50
We can simply reproduce this issue by this measure( Reproduced this issue only with https://github.com/gorhill/cronexpr source code, without heartbeat):
- created a simple unit test snippet:
package cronexpr
import (
"fmt"
"sync"
"testing"
"time"
)
var lock = &sync.Mutex{}
// First case uses to check the length of next month is shorter than current month, like August and September
func TestNextDayOfMonthFirstCase(t *testing.T) {
from, _ := time.Parse("2006-01-02 15:04:05", "2020-08-31 23:59:01")
parse := MustParse("6 * * * *")
// initialize actualDaysOfMonthList
parse.actualDaysOfMonthList = parse.calculateActualDaysOfMonth(from.Year(), int(time.August))
//// initialize i
go func(p *Expression) {
//lock.Lock()
//defer lock.Unlock()
fmt.Println(p.nextDayOfMonth(from))
}(parse)
go func(p *Expression) {
//lock.Lock()
//defer lock.Unlock()
p.actualDaysOfMonthList = p.calculateActualDaysOfMonth(from.Year(), int(time.September))
fmt.Println(p.actualDaysOfMonthList)
}(parse)
time.Sleep(time.Second * 5)
}
- In order to make this issue 100% reproducible, add the following sleep under this line:
time.Sleep(time.Second * 3)
Then, you will get the panic error like the following if you run this unittest:
=== RUN TestNextDayOfMonthFirstCase
[1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30]
panic: runtime error: index out of range [31] with length 30
goroutine 19 [running]:
github.com/gorhill/cronexpr.(*Expression).nextDayOfMonth(0xc000176000, 0x0, 0xed6df85c5, 0x0, 0x0, 0x0, 0x0)
/Users/fenxiao/ebay/project/go/src/github.com/gorhill/cronexpr/cronexpr_next.go:115 +0x395
github.com/gorhill/cronexpr.TestNextDayOfMonthFirstCase.func1(0x0, 0xed6df85c5, 0x0, 0xc000176000)
/Users/fenxiao/ebay/project/go/src/github.com/gorhill/cronexpr/cronexpr_next_test.go:22 +0x67
created by github.com/gorhill/cronexpr.TestNextDayOfMonthFirstCase
/Users/fenxiao/ebay/project/go/src/github.com/gorhill/cronexpr/cronexpr_next_test.go:19 +0x1a0
And if you uncomment my lock and unlock for this resource (calculateActualDaysOfMonth), you'll get rid of this issue.
Could you please help to investigate from Heartbeat side?
Thanks in advance.