"fatal error: concurrent map writes" or "fatal error: concurrent map read and map write" errors

[bitsofinfo]

It appears that when my helm does multiple sprig merge operations within a range I get a panic exiting w/ the error "fatal error: concurrent map writes"

Masterminds/sprig#120

[mumoshu]

Sounds like you're trying to mutate .Values or .Environment.Values with merge - right?

[bitsofinfo]

Well I'm using mergeOverwrite and the left side is a brand new dict so in theory no mutation. Trying to come up w/ a reproducible example.

[bitsofinfo]

worker 2/4 started
worker 3/4 started
worker 1/4 started
worker 1/4 finished
fatal error: concurrent map writes

goroutine 97 [running]:
runtime.throw(0x19b864b, 0x15)
        /usr/local/Cellar/go/1.12.5/libexec/src/runtime/panic.go:617 +0x72 fp=0xc000264250 sp=0xc000264220 pc=0x102db02
runtime.mapassign(0x188d280, 0xc000163200, 0xc000464400, 0xc000264310)
        /usr/local/Cellar/go/1.12.5/libexec/src/runtime/map.go:682 +0x5c8 fp=0xc0002642d8 sp=0xc000264250 pc=0x100fd48
reflect.mapassign(0x188d280, 0xc000163200, 0xc000464400, 0xc000464410)
        /usr/local/Cellar/go/1.12.5/libexec/src/runtime/map.go:1331 +0x3f fp=0xc000264308 sp=0xc0002642d8 pc=0x10117ef
reflect.Value.SetMapIndex(0x188d280, 0xc000163200, 0x15, 0x1844e40, 0xc000464400, 0x98, 0x1888fc0, 0xc000464410, 0x94)
        /usr/local/Cellar/go/1.12.5/libexec/src/reflect/value.go:1615 +0x234 fp=0xc000264370 sp=0xc000264308 pc=0x1088b04
github.com/imdario/mergo.deepMerge(0x188d280, 0xc000163200, 0x15, 0x188d280, 0xc000163200, 0x15, 0xc000265200, 0x5, 0xc000358700, 0x13, ...)
        /Users/dog1/go/pkg/mod/github.com/imdario/mergo@v0.3.6/merge.go:147 +0x677 fp=0xc0002645c0 sp=0xc000264370 pc=0x171f947
github.com/imdario/mergo.deepMerge(0x188d280, 0xc0001631d0, 0x15, 0x188d280, 0xc0001631d0, 0x15, 0xc000265200, 0x4, 0xc000358700, 0x0, ...)
        /Users/dog1/go/pkg/mod/github.com/imdario/mergo@v0.3.6/merge.go:115 +0x97d fp=0xc000264810 sp=0xc0002645c0 pc=0x171fc4d
github.com/imdario/mergo.deepMerge(0x188d280, 0xc00042d230, 0x15, 0x188d280, 0xc0001631a0, 0x15, 0xc000265200, 0x3, 0xc000358700, 0x19aed3f, ...)
        /Users/dog1/go/pkg/mod/github.com/imdario/mergo@v0.3.6/merge.go:115 +0x97d fp=0xc000264a60 sp=0xc000264810 pc=0x171fc4d
github.com/imdario/mergo.deepMerge(0x188d280, 0xc00042d200, 0x15, 0x188d280, 0xc000163170, 0x15, 0xc000265200, 0x2, 0xc000358700, 0x203000, ...)
        /Users/dog1/go/pkg/mod/github.com/imdario/mergo@v0.3.6/merge.go:115 +0x97d fp=0xc000264cb0 sp=0xc000264a60 pc=0x171fc4d
github.com/imdario/mergo.deepMerge(0x188d280, 0xc00042d170, 0x15, 0x188d280, 0xc000163140, 0x15, 0xc000265200, 0x1, 0xc000358700, 0x0, ...)
        /Users/dog1/go/pkg/mod/github.com/imdario/mergo@v0.3.6/merge.go:115 +0x97d fp=0xc000264f00 sp=0xc000264cb0 pc=0x171fc4d
github.com/imdario/mergo.deepMerge(0x188d280, 0xc000406038, 0x195, 0x188d280, 0xc000456240, 0x15, 0xc000265200, 0x0, 0xc000358700, 0x15, ...)
        /Users/dog1/go/pkg/mod/github.com/imdario/mergo@v0.3.6/merge.go:115 +0x97d fp=0xc000265150 sp=0xc000264f00 pc=0x171fc4d
github.com/imdario/mergo.merge(0x18274a0, 0xc000406038, 0x188d280, 0xc000456240, 0xc000215320, 0x1, 0x1, 0x0, 0x0)
        /Users/dog1/go/pkg/mod/github.com/imdario/mergo@v0.3.6/merge.go:251 +0x299 fp=0xc0002652d0 sp=0xc000265150 pc=0x1721d89
github.com/imdario/mergo.Merge(...)
        /Users/dog1/go/pkg/mod/github.com/imdario/mergo@v0.3.6/merge.go:206
github.com/roboll/helmfile/pkg/state.(*HelmState).Values(0xc00000c5a0, 0x1c, 0xc000215398, 0x10e903a)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/state/state_exec_tmpl.go:16 +0x139 fp=0xc000265340 sp=0xc0002652d0 pc=0x175e659
github.com/roboll/helmfile/pkg/state.(*HelmState).mustLoadVals(...)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/state/state_exec_tmpl.go:29
github.com/roboll/helmfile/pkg/state.(*HelmState).valuesFileTemplateData(0xc00000c5a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/state/state_exec_tmpl.go:40 +0x51 fp=0xc0002653a8 sp=0xc000265340 pc=0x175e7c1
github.com/roboll/helmfile/pkg/state.(*HelmState).RenderValuesFileToBytes(0xc00000c5a0, 0xc0001ded80, 0x28, 0x6, 0xc0001dec00, 0x28, 0xc0004641d0, 0x1)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/state/state.go:1285 +0x72 fp=0xc000265478 sp=0xc0002653a8 pc=0x1759e92
github.com/roboll/helmfile/pkg/state.(*HelmState).generateTemporaryValuesFiles(0xc00000c5a0, 0xc000452b00, 0x4, 0x4, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/state/state.go:1342 +0x2ea fp=0xc000265898 sp=0xc000265478 pc=0x175a9ea
github.com/roboll/helmfile/pkg/state.(*HelmState).namespaceAndValuesFlags(0xc00000c5a0, 0x1b5ce60, 0xc0001a6000, 0xc0004961e8, 0x4, 0xc0003e6700, 0x6, 0x8, 0x0, 0x0)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/state/state.go:1394 +0x17a fp=0xc000265c80 sp=0xc000265898 pc=0x175b91a
github.com/roboll/helmfile/pkg/state.(*HelmState).flagsForDiff(0xc00000c5a0, 0x1b5ce60, 0xc0001a6000, 0xc0004961e8, 0x4, 0x0, 0x0, 0x0, 0x0, 0x0)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/state/state.go:1254 +0x190 fp=0xc000265d28 sp=0xc000265c80 pc=0x17599b0
github.com/roboll/helmfile/pkg/state.(*HelmState).prepareDiffReleases.func2(0x4)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/state/state.go:701 +0x5a0 fp=0xc000265f00 sp=0xc000265d28 pc=0x1763a40
github.com/roboll/helmfile/pkg/state.(*HelmState).scatterGather.func1(0xc00000c5a0, 0xc00033f140, 0xc000392d70, 0xc00033f150, 0x4)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/state/state_run.go:41 +0x125 fp=0xc000265fb8 sp=0xc000265f00 pc=0x1765b75
runtime.goexit()
        /usr/local/Cellar/go/1.12.5/libexec/src/runtime/asm_amd64.s:1337 +0x1 fp=0xc000265fc0 sp=0xc000265fb8 pc=0x105a7b1
created by github.com/roboll/helmfile/pkg/state.(*HelmState).scatterGather
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/state/state_run.go:39 +0x1bb

goroutine 1 [chan receive]:
github.com/roboll/helmfile/pkg/state.(*HelmState).prepareDiffReleases.func3()
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/state/state.go:739 +0x1a4
github.com/roboll/helmfile/pkg/state.(*HelmState).scatterGather(0xc00000c5a0, 0x0, 0x3, 0xc0001625a0, 0xc000392d70, 0xc0005031a8)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/state/state_run.go:47 +0x1da
github.com/roboll/helmfile/pkg/state.(*HelmState).prepareDiffReleases(0xc00000c5a0, 0x1b5ce60, 0xc0001a6000, 0x21b2c88, 0x0, 0x0, 0x0, 0xc000500001, 0x1065804, 0x19c5b82, ...)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/state/state.go:686 +0x410
github.com/roboll/helmfile/pkg/state.(*HelmState).DiffReleases(0xc00000c5a0, 0x1b5ce60, 0xc0001a6000, 0x21b2c88, 0x0, 0x0, 0x0, 0x1, 0x1844e40, 0xc00052f810, ...)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/state/state.go:774 +0xab
github.com/roboll/helmfile/pkg/app.(*Run).Apply(0xc000162120, 0x1b58000, 0xc00007a720, 0x19298e0, 0x1973d01, 0xc000162120)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/app/run.go:139 +0x1fc
github.com/roboll/helmfile/pkg/app.(*App).Apply.func1(0xc000162120, 0xc000162120, 0xc000503d58, 0xc000503c20)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/app/app.go:127 +0x3d
github.com/roboll/helmfile/pkg/app.(*App).ForEachState.func1(0xc00000c5a0, 0x1b5ce60, 0xc0001a6000, 0xc000503e10, 0x0, 0x0)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/app/app.go:349 +0x96
github.com/roboll/helmfile/pkg/app.(*App).VisitDesiredStatesWithReleasesFiltered.func1(0xc00000c5a0, 0x1b5ce60, 0xc0001a6000, 0x0, 0xc00048a5a0, 0x1, 0xc00048a5a0)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/app/app.go:428 +0x52f
github.com/roboll/helmfile/pkg/app.(*App).visitStates.func1(0x7ffeefbff9d6, 0x19, 0xc00003e1e0, 0x53, 0x0, 0x0)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/app/app.go:327 +0x8ec
github.com/roboll/helmfile/pkg/app.(*App).visitStateFiles.func1(0x0, 0x0)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/app/app.go:215 +0x9d
github.com/roboll/helmfile/pkg/app.(*App).within(0xc0003b0000, 0x19a9334, 0x1, 0xc0000d79c0, 0xc0002667f8, 0x2)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/app/app.go:157 +0x725
github.com/roboll/helmfile/pkg/app.(*App).visitStateFiles(0xc0003b0000, 0x7ffeefbff9d6, 0x19, 0xc00009c3c0, 0x20, 0x18fbb40)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/app/app.go:209 +0x29f
github.com/roboll/helmfile/pkg/app.(*App).visitStates(0xc0003b0000, 0x7ffeefbff9d6, 0x19, 0xc00007a510, 0x1, 0x1, 0xc00007a740, 0x1, 0x1, 0x0, ...)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/app/app.go:250 +0xd4
github.com/roboll/helmfile/pkg/app.(*App).VisitDesiredStatesWithReleasesFiltered(0xc0003b0000, 0x7ffeefbff9d6, 0x19, 0xc00007a730, 0xc000266a01, 0xc00000ee40)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/app/app.go:398 +0x426
github.com/roboll/helmfile/pkg/app.(*App).ForEachState(0xc0003b0000, 0xc00000ee40, 0xc00007a720, 0x198d280)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/app/app.go:344 +0x81
github.com/roboll/helmfile/pkg/app.(*App).Apply(0xc0003b0000, 0x1b58000, 0xc00007a720, 0xc00007a720, 0x18d62e0)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/app/app.go:126 +0x6e
main.main.func8(0xc0003b0000, 0xc0003a8140, 0xc0000a6420, 0xc00007a700, 0x0)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/main.go:299 +0x6d
main.action.func1(0xc0003a8140, 0x0, 0x0)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/main.go:540 +0x121
reflect.Value.call(0x185cea0, 0xc00007a210, 0x13, 0x19a9d13, 0x4, 0xc000266e18, 0x1, 0x1, 0xc0000d6440, 0x1013d43, ...)
        /usr/local/Cellar/go/1.12.5/libexec/src/reflect/value.go:447 +0x461
reflect.Value.Call(0x185cea0, 0xc00007a210, 0x13, 0xc000266e18, 0x1, 0x1, 0xc0000ae680, 0xc0000ae6d8, 0x140)
        /usr/local/Cellar/go/1.12.5/libexec/src/reflect/value.go:308 +0xa4
github.com/urfave/cli.HandleAction(0x185cea0, 0xc00007a210, 0xc0003a8140, 0x0, 0x0)
        /Users/dog1/go/pkg/mod/github.com/urfave/cli@v0.0.0-20160620154522-6011f165dc28/app.go:483 +0x1ff
github.com/urfave/cli.Command.Run(0x19aa9cb, 0x5, 0x0, 0x0, 0x0, 0x0, 0x0, 0x19d3f07, 0x3f, 0x0, ...)
        /Users/dog1/go/pkg/mod/github.com/urfave/cli@v0.0.0-20160620154522-6011f165dc28/command.go:186 +0x8d1
github.com/urfave/cli.(*App).Run(0xc000001680, 0xc0000c0000, 0x12, 0x12, 0x0, 0x0)
        /Users/dog1/go/pkg/mod/github.com/urfave/cli@v0.0.0-20160620154522-6011f165dc28/app.go:237 +0x601
main.main()
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/main.go:393 +0x279c

goroutine 35 [select]:
go.opencensus.io/stats/view.(*worker).start(0xc0000d6740)
        /Users/dog1/go/pkg/mod/go.opencensus.io@v0.18.0/stats/view/worker.go:152 +0xdd
created by go.opencensus.io/stats/view.init.0
        /Users/dog1/go/pkg/mod/go.opencensus.io@v0.18.0/stats/view/worker.go:29 +0x57

goroutine 36 [syscall]:
os/signal.signal_recv(0x0)
        /usr/local/Cellar/go/1.12.5/libexec/src/runtime/sigqueue.go:139 +0x9f
os/signal.loop()
        /usr/local/Cellar/go/1.12.5/libexec/src/os/signal/signal_unix.go:23 +0x22
created by os/signal.init.0
        /usr/local/Cellar/go/1.12.5/libexec/src/os/signal/signal_unix.go:29 +0x41

goroutine 60 [chan receive]:
github.com/roboll/helmfile/pkg/app.(*App).visitStates.func1.1(0xc00009d980, 0xc0003b0000, 0xc00048a5a0)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/app/app.go:262 +0x52
created by github.com/roboll/helmfile/pkg/app.(*App).visitStates.func1
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/app/app.go:261 +0x2ba

goroutine 95 [runnable]:
github.com/roboll/helmfile/pkg/maputil.CastKeysToStrings(0x188d280, 0xc0004562d0, 0x70000c00052fc10, 0xffffffffffffffff, 0x0)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/maputil/maputil.go:26 +0x3d0
github.com/roboll/helmfile/pkg/maputil.recursivelyStringifyMapKey(0x188d280, 0xc0004562d0, 0xc000260db0, 0x0, 0x100000000000008, 0x15)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/maputil/maputil.go:42 +0x75
github.com/roboll/helmfile/pkg/maputil.CastKeysToStrings(0x188d280, 0xc0004562a0, 0x100000001088b04, 0xffffffffffffffff, 0x0)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/maputil/maputil.go:27 +0x40f
github.com/roboll/helmfile/pkg/maputil.recursivelyStringifyMapKey(0x188d280, 0xc0004562a0, 0xc000260fd8, 0x98, 0x1888fc0, 0xc00052fdf0)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/maputil/maputil.go:42 +0x75
github.com/roboll/helmfile/pkg/maputil.CastKeysToStrings(0x188d280, 0xc000456270, 0xc0001632f0, 0xc000261128, 0x1014922)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/maputil/maputil.go:27 +0x40f
github.com/roboll/helmfile/pkg/maputil.recursivelyStringifyMapKey(0x188d280, 0xc000456270, 0xc0000362b0, 0x9, 0xc000432628, 0x0)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/maputil/maputil.go:42 +0x75
github.com/roboll/helmfile/pkg/maputil.CastKeysToStrings(0x188d280, 0xc0001625d0, 0x188d280, 0xc000456240, 0xc00025b320)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/maputil/maputil.go:27 +0x40f
github.com/roboll/helmfile/pkg/state.(*HelmState).Values(0xc00000c5a0, 0x1c, 0xc00025b398, 0x10e903a)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/state/state_exec_tmpl.go:20 +0x169
github.com/roboll/helmfile/pkg/state.(*HelmState).mustLoadVals(...)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/state/state_exec_tmpl.go:29
github.com/roboll/helmfile/pkg/state.(*HelmState).valuesFileTemplateData(0xc00000c5a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/state/state_exec_tmpl.go:40 +0x51
github.com/roboll/helmfile/pkg/state.(*HelmState).RenderValuesFileToBytes(0xc00000c5a0, 0xc000434480, 0x28, 0x6, 0xc000434330, 0x28, 0xc00052fae0, 0x1)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/state/state.go:1285 +0x72
github.com/roboll/helmfile/pkg/state.(*HelmState).generateTemporaryValuesFiles(0xc00000c5a0, 0xc0003e8200, 0x4, 0x4, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/state/state.go:1342 +0x2ea
github.com/roboll/helmfile/pkg/state.(*HelmState).namespaceAndValuesFlags(0xc00000c5a0, 0x1b5ce60, 0xc0001a6000, 0xc0004965b8, 0x2, 0xc00043c080, 0x6, 0x8, 0x0, 0x0)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/state/state.go:1394 +0x17a
github.com/roboll/helmfile/pkg/state.(*HelmState).flagsForDiff(0xc00000c5a0, 0x1b5ce60, 0xc0001a6000, 0xc0004965b8, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/state/state.go:1254 +0x190
github.com/roboll/helmfile/pkg/state.(*HelmState).prepareDiffReleases.func2(0x2)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/state/state.go:701 +0x5a0
github.com/roboll/helmfile/pkg/state.(*HelmState).scatterGather.func1(0xc00000c5a0, 0xc00033f140, 0xc000392d70, 0xc00033f150, 0x2)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/state/state_run.go:41 +0x125
created by github.com/roboll/helmfile/pkg/state.(*HelmState).scatterGather
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/state/state_run.go:39 +0x1bb

goroutine 96 [runnable]:
github.com/roboll/helmfile/pkg/maputil.CastKeysToStrings(0x188d280, 0xc000163200, 0x300000000000015, 0xffffffffffffffff, 0x0)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/maputil/maputil.go:32 +0x463
github.com/roboll/helmfile/pkg/maputil.recursivelyStringifyMapKey(0x188d280, 0xc000163200, 0xc0004a0960, 0x206563616c706572, 0x2022312d312d2d22, 0x94)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/maputil/maputil.go:42 +0x75
github.com/roboll/helmfile/pkg/maputil.CastKeysToStrings(0x188d280, 0xc0001631d0, 0x2, 0xc00041c340, 0x10145c4)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/maputil/maputil.go:27 +0x40f
github.com/roboll/helmfile/pkg/maputil.recursivelyStringifyMapKey(0x188d280, 0xc0001631d0, 0xc0000370b0, 0xa, 0xc000198638, 0x0)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/maputil/maputil.go:42 +0x75
github.com/roboll/helmfile/pkg/maputil.CastKeysToStrings(0x188d280, 0xc00042d230, 0x50000c00032a300, 0xffffffffffffffff, 0x0)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/maputil/maputil.go:27 +0x40f
github.com/roboll/helmfile/pkg/maputil.recursivelyStringifyMapKey(0x188d280, 0xc00042d230, 0xc0004a0db0, 0x7, 0xc000198408, 0x0)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/maputil/maputil.go:42 +0x75
github.com/roboll/helmfile/pkg/maputil.CastKeysToStrings(0x188d280, 0xc00042d200, 0xc00032a240, 0xc0004a0f00, 0x1014922)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/maputil/maputil.go:27 +0x40f
github.com/roboll/helmfile/pkg/maputil.recursivelyStringifyMapKey(0x188d280, 0xc00042d200, 0xc000036fe0, 0xb, 0xc000198508, 0x0)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/maputil/maputil.go:42 +0x75
github.com/roboll/helmfile/pkg/maputil.CastKeysToStrings(0x188d280, 0xc00042d170, 0xc0001632f0, 0xffffffffffffffff, 0x0)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/maputil/maputil.go:27 +0x40f
github.com/roboll/helmfile/pkg/maputil.recursivelyStringifyMapKey(0x188d280, 0xc00042d170, 0xc0004a1200, 0x188d280, 0xc00021e000, 0x195)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/maputil/maputil.go:42 +0x75
github.com/roboll/helmfile/pkg/maputil.CastKeysToStrings(0x188d280, 0xc00032a090, 0x188d280, 0xc000456240, 0xc000211320)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/maputil/maputil.go:27 +0x40f
github.com/roboll/helmfile/pkg/state.(*HelmState).Values(0xc00000c5a0, 0x1c, 0xc000211398, 0x10e903a)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/state/state_exec_tmpl.go:20 +0x169
github.com/roboll/helmfile/pkg/state.(*HelmState).mustLoadVals(...)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/state/state_exec_tmpl.go:29
github.com/roboll/helmfile/pkg/state.(*HelmState).valuesFileTemplateData(0xc00000c5a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/state/state_exec_tmpl.go:40 +0x51
github.com/roboll/helmfile/pkg/state.(*HelmState).RenderValuesFileToBytes(0xc00000c5a0, 0xc0002181b0, 0x28, 0x6, 0xc000218060, 0x28, 0xc00045c070, 0x1)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/state/state.go:1285 +0x72
github.com/roboll/helmfile/pkg/state.(*HelmState).generateTemporaryValuesFiles(0xc00000c5a0, 0xc0000d6500, 0x4, 0x4, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/state/state.go:1342 +0x2ea
github.com/roboll/helmfile/pkg/state.(*HelmState).namespaceAndValuesFlags(0xc00000c5a0, 0x1b5ce60, 0xc0001a6000, 0xc0004963d0, 0x3, 0xc0001cc000, 0x6, 0x8, 0x0, 0x0)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/state/state.go:1394 +0x17a
github.com/roboll/helmfile/pkg/state.(*HelmState).flagsForDiff(0xc00000c5a0, 0x1b5ce60, 0xc0001a6000, 0xc0004963d0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/state/state.go:1254 +0x190
github.com/roboll/helmfile/pkg/state.(*HelmState).prepareDiffReleases.func2(0x3)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/state/state.go:701 +0x5a0
github.com/roboll/helmfile/pkg/state.(*HelmState).scatterGather.func1(0xc00000c5a0, 0xc00033f140, 0xc000392d70, 0xc00033f150, 0x3)
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/state/state_run.go:41 +0x125
created by github.com/roboll/helmfile/pkg/state.(*HelmState).scatterGather
        /Users/dog1/go/src/github.com/bitsofinfo/helmfile/pkg/state/state_run.go:39 +0x1bb```

[bitsofinfo]

(fyi the above is from my helmfile build from the PR I submitted yesterday)

The stack trace panic occurs AFTER all my releases are generated and printed to STDOUT (i.e. im in debug mode).

[bitsofinfo]

does it generate the debug output list of generated releases via a single thread, but then when processing for real, does it concurrently?

[bitsofinfo]

Ok here is a reproducible example: https://github.com/bitsofinfo/helmfile-737

[bitsofinfo]

Updated the example project, totally weird, especially the helmfile.fail3.yaml example (fails w/ panic even if I take out all merge and mergeOverwrite calls.

fatal error: concurrent map read and map write

[mumoshu]

Thanks! Examples for reproduction are always helpful.

This seems like happening due to that helmfile lazily and concurrently loads state values for rendering release values files.
In other words, any helmfile.yaml with 2 or more releases may be affected by this, at least from what I can see from code and stack traces:

fatal error: concurrent map read and map write

goroutine 41 [running]:
runtime.throw(0x19ba8dc, 0x21)
	/usr/local/Cellar/go/1.12.5/libexec/src/runtime/panic.go:617 +0x72 fp=0xc00047cbd0 sp=0xc00047cba0 pc=0x102b882
runtime.mapaccess2(0x1885440, 0xc0004b0630, 0xc000486140, 0xc000486140, 0xc000486160)
	/usr/local/Cellar/go/1.12.5/libexec/src/runtime/map.go:472 +0x284 fp=0xc00047cc18 sp=0xc00047cbd0 pc=0x100d2a4
reflect.mapaccess(0x1885440, 0xc0004b0630, 0xc000486140, 0x19b0ce5)
	/usr/local/Cellar/go/1.12.5/libexec/src/runtime/map.go:1321 +0x3f fp=0xc00047cc50 sp=0xc00047cc18 pc=0x100f4ff
reflect.Value.MapIndex(0x1885440, 0xc0004b0630, 0x15, 0x183d0c0, 0xc000486140, 0x98, 0x1881180, 0xc000486160, 0x94)
	/usr/local/Cellar/go/1.12.5/libexec/src/reflect/value.go:1112 +0x10a fp=0xc00047ccb0 sp=0xc00047cc50 pc=0x10843ba
github.com/imdario/mergo.deepMerge(0x1885440, 0xc0004b0630, 0x15, 0x1885440, 0xc0004b01e0, 0x15, 0xc00047d200, 0x1, 0xc0003be160, 0x0, ...)
	/Users/mumoshu/go/pkg/mod/github.com/imdario/mergo@v0.3.6/merge.go:90 +0x46f fp=0xc00047cf00 sp=0xc00047ccb0 pc=0x1719e2f
github.com/imdario/mergo.deepMerge(0x1885440, 0xc0000ca000, 0x195, 0x1885440, 0xc0004b0180, 0x15, 0xc00047d200, 0x0, 0xc0003be160, 0x15, ...)
	/Users/mumoshu/go/pkg/mod/github.com/imdario/mergo@v0.3.6/merge.go:115 +0x97d fp=0xc00047d150 sp=0xc00047cf00 pc=0x171a33d
github.com/imdario/mergo.merge(0x1820100, 0xc0000ca000, 0x1885440, 0xc0004b0180, 0xc00047d320, 0x1, 0x1, 0x0, 0x0)
	/Users/mumoshu/go/pkg/mod/github.com/imdario/mergo@v0.3.6/merge.go:251 +0x299 fp=0xc00047d2d0 sp=0xc00047d150 pc=0x171c479
github.com/imdario/mergo.Merge(...)
	/Users/mumoshu/go/pkg/mod/github.com/imdario/mergo@v0.3.6/merge.go:206
github.com/roboll/helmfile/pkg/state.(*HelmState).Values(0xc000284a00, 0xc, 0xc00047d398, 0x10e6bba)
	/Users/mumoshu/helmfile/pkg/state/state_exec_tmpl.go:22 +0x139 fp=0xc00047d340 sp=0xc00047d2d0 pc=0x1757b09
github.com/roboll/helmfile/pkg/state.(*HelmState).mustLoadVals(...)
	/Users/mumoshu/helmfile/pkg/state/state_exec_tmpl.go:37
github.com/roboll/helmfile/pkg/state.(*HelmState).valuesFileTemplateData(0xc000284a00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/Users/mumoshu/helmfile/pkg/state/state_exec_tmpl.go:48 +0x51 fp=0xc00047d3a8 sp=0xc00047d340 pc=0x1757c71
github.com/roboll/helmfile/pkg/state.(*HelmState).RenderValuesFileToBytes(0xc000284a00, 0xc0003ca160, 0x18, 0x6, 0xc0003ca040, 0x18, 0xc000486070, 0x1)
	/Users/mumoshu/helmfile/pkg/state/state.go:1289 +0x72 fp=0xc00047d478 sp=0xc00047d3a8 pc=0x1753342
github.com/roboll/helmfile/pkg/state.(*HelmState).generateTemporaryValuesFiles(0xc000284a00, 0xc0000a0200, 0x4, 0x4, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/Users/mumoshu/helmfile/pkg/state/state.go:1346 +0x2ea fp=0xc00047d898 sp=0xc00047d478 pc=0x1753e9a
github.com/roboll/helmfile/pkg/state.(*HelmState).namespaceAndValuesFlags(0xc000284a00, 0x1b54160, 0xc000465620, 0xc0003f31e8, 0x6, 0x21a39c8, 0x0, 0x0, 0x0, 0x0)
	/Users/mumoshu/helmfile/pkg/state/state.go:1398 +0x17a fp=0xc00047dc80 sp=0xc00047d898 pc=0x1754dca
github.com/roboll/helmfile/pkg/state.(*HelmState).flagsForDiff(0xc000284a00, 0x1b54160, 0xc000465620, 0xc0003f31e8, 0x6, 0x0, 0x0, 0x0, 0x0, 0x0)
	/Users/mumoshu/helmfile/pkg/state/state.go:1258 +0x190 fp=0xc00047dd28 sp=0xc00047dc80 pc=0x1752e60
github.com/roboll/helmfile/pkg/state.(*HelmState).prepareDiffReleases.func2(0x6)
	/Users/mumoshu/helmfile/pkg/state/state.go:705 +0x5a0 fp=0xc00047df00 sp=0xc00047dd28 pc=0x175cef0
github.com/roboll/helmfile/pkg/state.(*HelmState).scatterGather.func1(0xc000284a00, 0xc0000360a0, 0xc0000d0190, 0xc0000360b0, 0x6)
	/Users/mumoshu/helmfile/pkg/state/state_run.go:41 +0x125 fp=0xc00047dfb8 sp=0xc00047df00 pc=0x175f025
runtime.goexit()
	/usr/local/Cellar/go/1.12.5/libexec/src/runtime/asm_amd64.s:1337 +0x1 fp=0xc00047dfc0 sp=0xc00047dfb8 pc=0x1058291
created by github.com/roboll/helmfile/pkg/state.(*HelmState).scatterGather
	/Users/mumoshu/helmfile/pkg/state/state_run.go:39 +0x1bb

goroutine 1 [chan receive]:

[bitsofinfo]

i see the CI test failed, is this something i could run on my own?

[bitsofinfo]

tried building locally and running, still getting the issue

[mumoshu]

@bitsofinfo Thanks for testing!
In which example are you still seeing the issue?
Would you mind sharing full log or at least the stack trace you're seeing, so that I can see where the problem is?

[bitsofinfo]

Sorry here you go

Running https://github.com/bitsofinfo/helmfile-737

helmfile --log-level debug -f helmfile.fail.yaml -e test4 apply

processing file "helmfile.fail.yaml" in directory "."
first-pass rendering starting for "helmfile.fail.yaml.part.0": inherited=&{test4 map[] map[]}, overrode=<nil>
first-pass uses: &{test4 map[] map[]}
first-pass produced: &{test4 map[chartConfigs:map[name:test4yahoo prop2:something] environments:map[stage:map[contexts:map[context1:map[services:[map[chartConfigs:map[appdeploy:map[chartValues:map[baseValues:[test3 test4] baseValuesRootDir:.]] items:map[p1:one p2:\{\{ .service.version | replace "--1-1" "" \}\}] name:test4-1] hooks:map[hook1:test] name:test4-service-1 version:8] map[name:test4-service-2] map[chartConfigs:map[name:test4-3 prop2:something-2] name:test4-service-3] map[name:test4-service-4] map[chartConfigs:map[items:map[p1:one p2:two] name:test4-5] name:test4-service-5] map[name:test4-service-6]]]]]] someProperty:test4 someProperty2:test4prop2] map[chartConfigs:map[appdeploy:map[chartValues:map[baseValues:[test1 test2] baseValuesRootDir:.]] items:map[p1:one p2:two] name:default prop2:default] someProp1:default]}
first-pass rendering result of "helmfile.fail.yaml.part.0": {test4 map[chartConfigs:map[name:test4yahoo prop2:something] environments:map[stage:map[contexts:map[context1:map[services:[map[chartConfigs:map[appdeploy:map[chartValues:map[baseValues:[test3 test4] baseValuesRootDir:.]] items:map[p1:one p2:\{\{ .service.version | replace "--1-1" "" \}\}] name:test4-1] hooks:map[hook1:test] name:test4-service-1 version:8] map[name:test4-service-2] map[chartConfigs:map[name:test4-3 prop2:something-2] name:test4-service-3] map[name:test4-service-4] map[chartConfigs:map[items:map[p1:one p2:two] name:test4-5] name:test4-service-5] map[name:test4-service-6]]]]]] someProperty:test4 someProperty2:test4prop2] map[chartConfigs:map[appdeploy:map[chartValues:map[baseValues:[test1 test2] baseValuesRootDir:.]] items:map[p1:one p2:two] name:default prop2:default] someProp1:default]}
vals:
map[chartConfigs:map[appdeploy:map[chartValues:map[baseValues:[test1 test2] baseValuesRootDir:.]] items:map[p1:one p2:two] name:test4yahoo prop2:something] environments:map[stage:map[contexts:map[context1:map[services:[map[chartConfigs:map[appdeploy:map[chartValues:map[baseValues:[test3 test4] baseValuesRootDir:.]] items:map[p1:one p2:\{\{ .service.version | replace "--1-1" "" \}\}] name:test4-1] hooks:map[hook1:test] name:test4-service-1 version:8] map[name:test4-service-2] map[chartConfigs:map[name:test4-3 prop2:something-2] name:test4-service-3] map[name:test4-service-4] map[chartConfigs:map[items:map[p1:one p2:two] name:test4-5] name:test4-service-5] map[name:test4-service-6]]]]]] someProp1:default someProperty:test4 someProperty2:test4prop2]
defaultVals:[map[chartConfigs:map[appdeploy:map[chartValues:map[baseValues:[test1 test2] baseValuesRootDir:.]] items:map[p1:one p2:two] name:default prop2:default] someProp1:default]]
second-pass rendering result of "helmfile.fail.yaml.part.0":
 0: values:
 1:   - someProp1: "default"
 2:     chartConfigs:
 3:       name: "default"
 4:       prop2: "default"
 5:       items:
 6:         p1: "one"
 7:         p2: "two"
 8:       appdeploy:
 9:         chartValues:
10:           baseValuesRootDir: "."
11:           baseValues:
12:             - test1
13:             - test2
14:
15: environments:
16:
17:
18:   test4:
19:     values:
20:       - someProperty: "test4"
21:         someProperty2: "test4prop2"
22:         chartConfigs:
23:           name: "test4yahoo"
24:           prop2: "something"
25:         environments:
26:           stage:
27:             contexts:
28:               context1:
29:                 services:
30:                   - name: "test4-service-1"
31:                     version: 8
32:                     hooks:
33:                       hook1: "test"
34:                     chartConfigs:
35:                       name: "test4-1"
36:                       items:
37:                         p1: "one"
38:                         p2: "\\{\\{ .service.version | replace \"--1-1\" \"\" \\}\\}"
39:                       appdeploy:
40:                         chartValues:
41:                           baseValuesRootDir: "."
42:                           baseValues:
43:                             - test3
44:                             - test4
45:                   - name: "test4-service-2"
46:                   - name: "test4-service-3"
47:                     chartConfigs:
48:                       name: "test4-3"
49:                       prop2: "something-2"
50:                   - name: "test4-service-4"
51:                   - name: "test4-service-5"
52:                     chartConfigs:
53:                       name: "test4-5"
54:                       items:
55:                         p1: "one"
56:                         p2: "two"
57:                   - name: "test4-service-6"
58:
59: releases:
60:
61:
62:
63:
64:
65:
66:
67:
68:   - name: nginx-context1-test4-service-1
69:     chart: stable/nginx-ingress
70:     values:
71:
72:
73:       - ./values/test3/values.yaml
74:
75:       - ./values/test4/values.yaml
76:
77:
78:       - chartConfigName: context1-test4-1
79:         prop2: something
80:         env1: test4
81:         state1: default
82:
83:
84:         p1: one
85:         p12: one
86:
87:
88:         p2: \{\{ .service.version | replace "--1-1" "" \}\}
89:         p22: \{\{ .service.version | replace "--1-1" "" \}\}
90:
91:
92:         someProperty2: test4prop2
93:
94:         someProperty2Again: test4prop2
95:
96:
97:
98:
99:
100:
101:
102:
103:
104:   - name: nginx-context1-test4-service-2
105:     chart: stable/nginx-ingress
106:     values:
107:
108:
109:       - ./values/test3/values.yaml
110:
111:       - ./values/test4/values.yaml
112:
113:
114:       - chartConfigName: context1-test4yahoo
115:         prop2: something
116:         env1: test4
117:         state1: default
118:
119:
120:         p1: one
121:         p12: one
122:
123:
124:         p2: \{\{ .service.version | replace "--1-1" "" \}\}
125:         p22: \{\{ .service.version | replace "--1-1" "" \}\}
126:
127:
128:         someProperty2: test4prop2
129:
130:         someProperty2Again: test4prop2
131:
132:
133:
134:
135:
136:
137:
138:
139:
140:   - name: nginx-context1-test4-service-3
141:     chart: stable/nginx-ingress
142:     values:
143:
144:
145:       - ./values/test3/values.yaml
146:
147:       - ./values/test4/values.yaml
148:
149:
150:       - chartConfigName: context1-test4-3
151:         prop2: something-2
152:         env1: test4
153:         state1: default
154:
155:
156:         p1: one
157:         p12: one
158:
159:
160:         p2: \{\{ .service.version | replace "--1-1" "" \}\}
161:         p22: \{\{ .service.version | replace "--1-1" "" \}\}
162:
163:
164:         someProperty2: test4prop2
165:
166:         someProperty2Again: test4prop2
167:
168:
169:
170:
171:
172:
173:
174:
175:
176:   - name: nginx-context1-test4-service-4
177:     chart: stable/nginx-ingress
178:     values:
179:
180:
181:       - ./values/test3/values.yaml
182:
183:       - ./values/test4/values.yaml
184:
185:
186:       - chartConfigName: context1-test4yahoo
187:         prop2: something
188:         env1: test4
189:         state1: default
190:
191:
192:         p1: one
193:         p12: one
194:
195:
196:         p2: \{\{ .service.version | replace "--1-1" "" \}\}
197:         p22: \{\{ .service.version | replace "--1-1" "" \}\}
198:
199:
200:         someProperty2: test4prop2
201:
202:         someProperty2Again: test4prop2
203:
204:
205:
206:
207:
208:
209:
210:
211:
212:   - name: nginx-context1-test4-service-5
213:     chart: stable/nginx-ingress
214:     values:
215:
216:
217:       - ./values/test3/values.yaml
218:
219:       - ./values/test4/values.yaml
220:
221:
222:       - chartConfigName: context1-test4-5
223:         prop2: something
224:         env1: test4
225:         state1: default
226:
227:
228:         p1: one
229:         p12: one
230:
231:
232:         p2: two
233:         p22: two
234:
235:
236:         someProperty2: test4prop2
237:
238:         someProperty2Again: test4prop2
239:
240:
241:
242:
243:
244:
245:
246:
247:
248:   - name: nginx-context1-test4-service-6
249:     chart: stable/nginx-ingress
250:     values:
251:
252:
253:       - ./values/test3/values.yaml
254:
255:       - ./values/test4/values.yaml
256:
257:
258:       - chartConfigName: context1-test4yahoo
259:         prop2: something
260:         env1: test4
261:         state1: default
262:
263:
264:         p1: one
265:         p12: one
266:
267:
268:         p2: two
269:         p22: two
270:
271:
272:         someProperty2: test4prop2
273:
274:         someProperty2Again: test4prop2
275:
276:
277:
278:

merged environment: &{test4 map[chartConfigs:map[name:test4yahoo prop2:something] environments:map[stage:map[contexts:map[context1:map[services:[map[chartConfigs:map[appdeploy:map[chartValues:map[baseValues:[test3 test4] baseValuesRootDir:.]] items:map[p1:one p2:\{\{ .service.version | replace "--1-1" "" \}\}] name:test4-1] hooks:map[hook1:test] name:test4-service-1 version:8] map[name:test4-service-2] map[chartConfigs:map[name:test4-3 prop2:something-2] name:test4-service-3] map[name:test4-service-4] map[chartConfigs:map[items:map[p1:one p2:two] name:test4-5] name:test4-service-5] map[name:test4-service-6]]]]]] someProperty:test4 someProperty2:test4prop2] map[chartConfigs:map[appdeploy:map[chartValues:map[baseValues:[test1 test2] baseValuesRootDir:.]] items:map[p1:one p2:two] name:default prop2:default] someProp1:default]}
worker 6/6 started
worker 2/6 started
worker 4/6 started
worker 3/6 started
worker 5/6 started
fatal error: concurrent map writes
fatal error: concurrent map writes
fatal error: concurrent map writes

goroutine 85 [running]:
runtime.throw(0x19b864b, 0x15)
	/usr/local/Cellar/go/1.12.5/libexec/src/runtime/panic.go:617 +0x72 fp=0xc0001e6b90 sp=0xc0001e6b60 pc=0x102db02
runtime.mapassign(0x188d280, 0xc00040dfb0, 0xc0000b6490, 0xc0001e6c50)
	/usr/local/Cellar/go/1.12.5/libexec/src/runtime/map.go:682 +0x5c8 fp=0xc0001e6c18 sp=0xc0001e6b90 pc=0x100fd48
reflect.mapassign(0x188d280, 0xc00040dfb0, 0xc0000b6490, 0xc0000b64c0)
	/usr/local/Cellar/go/1.12.5/libexec/src/runtime/map.go:1331 +0x3f fp=0xc0001e6c48 sp=0xc0001e6c18 pc=0x10117ef
reflect.Value.SetMapIndex(0x188d280, 0xc00040dfb0, 0x15, 0x1844e40, 0xc0000b6490, 0x98, 0x1888fc0, 0xc0000b64c0, 0x94)
	/usr/local/Cellar/go/1.12.5/libexec/src/reflect/value.go:1615 +0x234 fp=0xc0001e6cb0 sp=0xc0001e6c48 pc=0x1088b04
github.com/imdario/mergo.deepMerge(0x188d280, 0xc00040dfb0, 0x15, 0x188d280, 0xc00040d7a0, 0x15, 0xc0001e7200, 0x1, 0xc0001ee060, 0x0, ...)
	/Users/dog/go/pkg/mod/github.com/imdario/mergo@v0.3.6/merge.go:147 +0x677 fp=0xc0001e6f00 sp=0xc0001e6cb0 pc=0x171f947
github.com/imdario/mergo.deepMerge(0x188d280, 0xc0001f0000, 0x195, 0x188d280, 0xc00040d740, 0x15, 0xc0001e7200, 0x0, 0xc0001ee060, 0x15, ...)
	/Users/dog/go/pkg/mod/github.com/imdario/mergo@v0.3.6/merge.go:115 +0x97d fp=0xc0001e7150 sp=0xc0001e6f00 pc=0x171fc4d
github.com/imdario/mergo.merge(0x18274a0, 0xc0001f0000, 0x188d280, 0xc00040d740, 0xc0001e7320, 0x1, 0x1, 0x0, 0x0)
	/Users/dog/go/pkg/mod/github.com/imdario/mergo@v0.3.6/merge.go:251 +0x299 fp=0xc0001e72d0 sp=0xc0001e7150 pc=0x1721d89
github.com/imdario/mergo.Merge(...)
	/Users/dog/go/pkg/mod/github.com/imdario/mergo@v0.3.6/merge.go:206
github.com/roboll/helmfile/pkg/state.(*HelmState).Values(0xc0000c41e0, 0xc, 0xc0001e7398, 0x10e903a)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state_exec_tmpl.go:16 +0x139 fp=0xc0001e7340 sp=0xc0001e72d0 pc=0x175e659
github.com/roboll/helmfile/pkg/state.(*HelmState).mustLoadVals(...)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state_exec_tmpl.go:29
github.com/roboll/helmfile/pkg/state.(*HelmState).valuesFileTemplateData(0xc0000c41e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state_exec_tmpl.go:40 +0x51 fp=0xc0001e73a8 sp=0xc0001e7340 pc=0x175e7c1
github.com/roboll/helmfile/pkg/state.(*HelmState).RenderValuesFileToBytes(0xc0000c41e0, 0xc00050a100, 0x18, 0x6, 0xc00050a040, 0x18, 0xc0000b6260, 0x1)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state.go:1285 +0x72 fp=0xc0001e7478 sp=0xc0001e73a8 pc=0x1759e92
github.com/roboll/helmfile/pkg/state.(*HelmState).generateTemporaryValuesFiles(0xc0000c41e0, 0xc0003d8040, 0x3, 0x4, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state.go:1342 +0x2ea fp=0xc0001e7898 sp=0xc0001e7478 pc=0x175a9ea
github.com/roboll/helmfile/pkg/state.(*HelmState).namespaceAndValuesFlags(0xc0000c41e0, 0x1b5ce60, 0xc00048e000, 0xc0001b11b8, 0x3, 0x21b2c88, 0x0, 0x0, 0x0, 0x0)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state.go:1394 +0x17a fp=0xc0001e7c80 sp=0xc0001e7898 pc=0x175b91a
github.com/roboll/helmfile/pkg/state.(*HelmState).flagsForDiff(0xc0000c41e0, 0x1b5ce60, 0xc00048e000, 0xc0001b11b8, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state.go:1254 +0x190 fp=0xc0001e7d28 sp=0xc0001e7c80 pc=0x17599b0
github.com/roboll/helmfile/pkg/state.(*HelmState).prepareDiffReleases.func2(0x3)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state.go:701 +0x5a0 fp=0xc0001e7f00 sp=0xc0001e7d28 pc=0x1763a40
github.com/roboll/helmfile/pkg/state.(*HelmState).scatterGather.func1(0xc0000c41e0, 0xc000036e38, 0xc0000e0fa0, 0xc000036e60, 0x3)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state_run.go:41 +0x125 fp=0xc0001e7fb8 sp=0xc0001e7f00 pc=0x1765b75
runtime.goexit()
	/usr/local/Cellar/go/1.12.5/libexec/src/runtime/asm_amd64.s:1337 +0x1 fp=0xc0001e7fc0 sp=0xc0001e7fb8 pc=0x105a7b1
created by github.com/roboll/helmfile/pkg/state.(*HelmState).scatterGather
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state_run.go:39 +0x1bb

goroutine 1 [chan receive]:
github.com/roboll/helmfile/pkg/state.(*HelmState).prepareDiffReleases.func3()
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state.go:739 +0x1a4
github.com/roboll/helmfile/pkg/state.(*HelmState).scatterGather(0xc0000c41e0, 0x0, 0x6, 0xc000486a80, 0xc0000e0fa0, 0xc0005b31a8)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state_run.go:47 +0x1da
github.com/roboll/helmfile/pkg/state.(*HelmState).prepareDiffReleases(0xc0000c41e0, 0x1b5ce60, 0xc00048e000, 0x21b2c88, 0x0, 0x0, 0x0, 0xc0005b0001, 0x1065804, 0x19c5b82, ...)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state.go:686 +0x410
github.com/roboll/helmfile/pkg/state.(*HelmState).DiffReleases(0xc0000c41e0, 0x1b5ce60, 0xc00048e000, 0x21b2c88, 0x0, 0x0, 0x0, 0x1, 0xc0000351a0, 0x0, ...)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state.go:774 +0xab
github.com/roboll/helmfile/pkg/app.(*Run).Apply(0xc000486510, 0x1b58000, 0xc0001d8870, 0x19298e0, 0x1973d01, 0xc000486510)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/app/run.go:139 +0x1fc
github.com/roboll/helmfile/pkg/app.(*App).Apply.func1(0xc000486510, 0xc000486510, 0xc0005b3d98, 0xc0005b3c20)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/app/app.go:127 +0x3d
github.com/roboll/helmfile/pkg/app.(*App).ForEachState.func1(0xc0000c41e0, 0x1b5ce60, 0xc00048e000, 0xc0005b3e20, 0x0, 0x0)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/app/app.go:349 +0x96
github.com/roboll/helmfile/pkg/app.(*App).VisitDesiredStatesWithReleasesFiltered.func1(0xc0000c41e0, 0x1b5ce60, 0xc00048e000, 0x0, 0xc00000c960, 0x1, 0xc00000c960)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/app/app.go:428 +0x52f
github.com/roboll/helmfile/pkg/app.(*App).visitStates.func1(0x7ffeefbffa22, 0x12, 0xc00003c230, 0x44, 0x0, 0x0)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/app/app.go:327 +0x8ec
github.com/roboll/helmfile/pkg/app.(*App).visitStateFiles.func1(0x0, 0x0)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/app/app.go:215 +0x9d
github.com/roboll/helmfile/pkg/app.(*App).within(0xc0003e0000, 0x19a9334, 0x1, 0xc0000cb580, 0xc0002647f8, 0x2)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/app/app.go:157 +0x725
github.com/roboll/helmfile/pkg/app.(*App).visitStateFiles(0xc0003e0000, 0x7ffeefbffa22, 0x12, 0xc0000a4780, 0x20, 0x18fbb40)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/app/app.go:209 +0x29f
github.com/roboll/helmfile/pkg/app.(*App).visitStates(0xc0003e0000, 0x7ffeefbffa22, 0x12, 0x21b2c88, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/app/app.go:250 +0xd4
github.com/roboll/helmfile/pkg/app.(*App).VisitDesiredStatesWithReleasesFiltered(0xc0003e0000, 0x7ffeefbffa22, 0x12, 0xc0001d8880, 0xc000264a01, 0xc00000f340)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/app/app.go:398 +0x426
github.com/roboll/helmfile/pkg/app.(*App).ForEachState(0xc0003e0000, 0xc00000f340, 0xc0001d8870, 0x198d280)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/app/app.go:344 +0x81
github.com/roboll/helmfile/pkg/app.(*App).Apply(0xc0003e0000, 0x1b58000, 0xc0001d8870, 0xc0001d8870, 0x18d62e0)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/app/app.go:126 +0x6e
main.main.func8(0xc0003e0000, 0xc00011eb40, 0x0, 0xc0001d8850, 0x0)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/main.go:299 +0x6d
main.action.func1(0xc00011eb40, 0x0, 0x0)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/main.go:540 +0x121
reflect.Value.call(0x185cea0, 0xc0001d8410, 0x13, 0x19a9d13, 0x4, 0xc000264e18, 0x1, 0x1, 0xc000043240, 0x1013d43, ...)
	/usr/local/Cellar/go/1.12.5/libexec/src/reflect/value.go:447 +0x461
reflect.Value.Call(0x185cea0, 0xc0001d8410, 0x13, 0xc000264e18, 0x1, 0x1, 0xc0000eb040, 0xc0000eb098, 0x140)
	/usr/local/Cellar/go/1.12.5/libexec/src/reflect/value.go:308 +0xa4
github.com/urfave/cli.HandleAction(0x185cea0, 0xc0001d8410, 0xc00011eb40, 0x0, 0x0)
	/Users/dog/go/pkg/mod/github.com/urfave/cli@v0.0.0-20160620154522-6011f165dc28/app.go:483 +0x1ff
github.com/urfave/cli.Command.Run(0x19aa9cb, 0x5, 0x0, 0x0, 0x0, 0x0, 0x0, 0x19d3f07, 0x3f, 0x0, ...)
	/Users/dog/go/pkg/mod/github.com/urfave/cli@v0.0.0-20160620154522-6011f165dc28/command.go:186 +0x8d1
github.com/urfave/cli.(*App).Run(0xc000001500, 0xc000032080, 0x8, 0x8, 0x0, 0x0)
	/Users/dog/go/pkg/mod/github.com/urfave/cli@v0.0.0-20160620154522-6011f165dc28/app.go:237 +0x601
main.main()
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/main.go:393 +0x279c

goroutine 35 [select]:
go.opencensus.io/stats/view.(*worker).start(0xc0000ca300)
	/Users/dog/go/pkg/mod/go.opencensus.io@v0.18.0/stats/view/worker.go:152 +0xdd
created by go.opencensus.io/stats/view.init.0
	/Users/dog/go/pkg/mod/go.opencensus.io@v0.18.0/stats/view/worker.go:29 +0x57

goroutine 36 [syscall]:
os/signal.signal_recv(0x0)
	/usr/local/Cellar/go/1.12.5/libexec/src/runtime/sigqueue.go:139 +0x9f
os/signal.loop()
	/usr/local/Cellar/go/1.12.5/libexec/src/os/signal/signal_unix.go:23 +0x22
created by os/signal.init.0
	/usr/local/Cellar/go/1.12.5/libexec/src/os/signal/signal_unix.go:29 +0x41

goroutine 22 [chan receive]:
github.com/roboll/helmfile/pkg/app.(*App).visitStates.func1.1(0xc0004676e0, 0xc0003e0000, 0xc00000c960)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/app/app.go:262 +0x52
created by github.com/roboll/helmfile/pkg/app.(*App).visitStates.func1
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/app/app.go:261 +0x2ba

goroutine 83 [runnable]:
internal/poll.runtime_Semacquire(0xc0000a40cc)
	/usr/local/Cellar/go/1.12.5/libexec/src/runtime/sema.go:61 +0x39
internal/poll.(*fdMutex).rwlock(0xc0000a40c0, 0x1984300, 0xc00000e100)
	/usr/local/Cellar/go/1.12.5/libexec/src/internal/poll/fd_mutex.go:154 +0xad
internal/poll.(*FD).writeLock(...)
	/usr/local/Cellar/go/1.12.5/libexec/src/internal/poll/fd_mutex.go:239
internal/poll.(*FD).Write(0xc0000a40c0, 0xc00043a400, 0x13, 0x400, 0x0, 0x0, 0x0)
	/usr/local/Cellar/go/1.12.5/libexec/src/internal/poll/fd_unix.go:255 +0x4e
os.(*File).write(...)
	/usr/local/Cellar/go/1.12.5/libexec/src/os/file_unix.go:280
os.(*File).Write(0xc000010018, 0xc00043a400, 0x13, 0x400, 0x2195f00, 0x0, 0x0)
	/usr/local/Cellar/go/1.12.5/libexec/src/os/file.go:145 +0x76
go.uber.org/zap/zapcore.(*ioCore).Write(0xc0000db890, 0xff, 0xbf3f7363a6277c90, 0x50997b9, 0x2195f00, 0x0, 0x0, 0xc000414040, 0x12, 0x0, ...)
	/Users/dog/go/pkg/mod/go.uber.org/zap@v1.8.0/zapcore/core.go:90 +0x107
go.uber.org/zap/zapcore.(*CheckedEntry).Write(0xc0003c00b0, 0x0, 0x0, 0x0)
	/Users/dog/go/pkg/mod/go.uber.org/zap@v1.8.0/zapcore/entry.go:215 +0x119
go.uber.org/zap.(*SugaredLogger).log(0xc0000100a0, 0xc00041a0ff, 0x19b7e46, 0x14, 0xc0001fa788, 0x2, 0x2, 0x0, 0x0, 0x0)
	/Users/dog/go/pkg/mod/go.uber.org/zap@v1.8.0/sugar.go:234 +0x101
go.uber.org/zap.(*SugaredLogger).Debugf(...)
	/Users/dog/go/pkg/mod/go.uber.org/zap@v1.8.0/sugar.go:133
github.com/roboll/helmfile/pkg/state.(*HelmState).scatterGather.func1(0xc0000c41e0, 0xc000036e38, 0xc0000e0fa0, 0xc000036e60, 0x1)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state_run.go:40 +0x10c
created by github.com/roboll/helmfile/pkg/state.(*HelmState).scatterGather
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state_run.go:39 +0x1bb

goroutine 84 [running]:
	goroutine running on other thread; stack unavailable
created by github.com/roboll/helmfile/pkg/state.(*HelmState).scatterGather
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state_run.go:39 +0x1bb

goroutine 86 [runnable]:
syscall.syscall(0x10a6b00, 0xc000432000, 0x1000000, 0x0, 0x5, 0x0, 0x0)
	/usr/local/Cellar/go/1.12.5/libexec/src/runtime/sys_darwin.go:63 +0x2e
syscall.Open(0xc0003ca140, 0x18, 0x1000000, 0x0, 0xc0001f2180, 0x10e53a9, 0xc0003ca140)
	/usr/local/Cellar/go/1.12.5/libexec/src/syscall/zsyscall_darwin_amd64.go:1086 +0xaf
os.openFileNolog(0xc0003ca140, 0x18, 0x0, 0x0, 0x1844e40, 0x1, 0xc0001f2180)
	/usr/local/Cellar/go/1.12.5/libexec/src/os/file_unix.go:201 +0xaf
os.OpenFile(0xc0003ca140, 0x18, 0x0, 0x0, 0x13, 0x0, 0x1)
	/usr/local/Cellar/go/1.12.5/libexec/src/os/file.go:284 +0x5f
os.Open(...)
	/usr/local/Cellar/go/1.12.5/libexec/src/os/file.go:265
io/ioutil.ReadFile(0xc0003ca140, 0x18, 0x0, 0x0, 0x0, 0x0, 0x0)
	/usr/local/Cellar/go/1.12.5/libexec/src/io/ioutil/ioutil.go:53 +0x64
github.com/roboll/helmfile/pkg/tmpl.(*FileRenderer).RenderToBytes(0xc000427410, 0xc0003ca140, 0x18, 0xc00042e040, 0xc00040df50, 0x0, 0x0, 0xc0000b88a0)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/tmpl/file_renderer.go:62 +0xe4
github.com/roboll/helmfile/pkg/state.(*HelmState).RenderValuesFileToBytes(0xc0000c41e0, 0xc0003ca140, 0x18, 0x6, 0xc0003ca080, 0x18, 0xc00007a0a0, 0x1)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state.go:1286 +0x17f
github.com/roboll/helmfile/pkg/state.(*HelmState).generateTemporaryValuesFiles(0xc0000c41e0, 0xc0000d6080, 0x3, 0x4, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state.go:1342 +0x2ea
github.com/roboll/helmfile/pkg/state.(*HelmState).namespaceAndValuesFlags(0xc0000c41e0, 0x1b5ce60, 0xc00048e000, 0xc0001b0fd0, 0x4, 0x21b2c88, 0x0, 0x0, 0x0, 0x0)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state.go:1394 +0x17a
github.com/roboll/helmfile/pkg/state.(*HelmState).flagsForDiff(0xc0000c41e0, 0x1b5ce60, 0xc00048e000, 0xc0001b0fd0, 0x4, 0x0, 0x0, 0x0, 0x0, 0x0)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state.go:1254 +0x190
github.com/roboll/helmfile/pkg/state.(*HelmState).prepareDiffReleases.func2(0x4)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state.go:701 +0x5a0
github.com/roboll/helmfile/pkg/state.(*HelmState).scatterGather.func1(0xc0000c41e0, 0xc000036e38, 0xc0000e0fa0, 0xc000036e60, 0x4)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state_run.go:41 +0x125
created by github.com/roboll/helmfile/pkg/state.(*HelmState).scatterGather
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state_run.go:39 +0x1bb

goroutine 87 [running]:
	goroutine running on other thread; stack unavailable
created by github.com/roboll/helmfile/pkg/state.(*HelmState).scatterGather
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state_run.go:39 +0x1bb

goroutine 88 [runnable]:
github.com/roboll/helmfile/pkg/maputil.CastKeysToStrings(0x188d280, 0xc00040d8c0, 0x400000000000000, 0xc0001ea5d0, 0x10145c4)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/maputil/maputil.go:6 +0x37
github.com/roboll/helmfile/pkg/maputil.recursivelyStringifyMapKey(0x188d280, 0xc00040d8c0, 0xc0000c9820, 0x7, 0xc0000e6cf8, 0x0)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/maputil/maputil.go:42 +0x75
github.com/roboll/helmfile/pkg/maputil.CastKeysToStrings(0x188d280, 0xc00040d890, 0x0, 0x0, 0xc0001ea838)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/maputil/maputil.go:27 +0x40f
github.com/roboll/helmfile/pkg/maputil.recursivelyStringifyMapKey(0x188d280, 0xc00040d890, 0x300000000000000, 0xffffffffffffffff, 0x0, 0x0)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/maputil/maputil.go:42 +0x75
github.com/roboll/helmfile/pkg/maputil.recursivelyStringifyMapKey(0x1834e20, 0xc00000fe20, 0xc0001ea960, 0x0, 0x0, 0x0)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/maputil/maputil.go:50 +0x1ae
github.com/roboll/helmfile/pkg/maputil.CastKeysToStrings(0x188d280, 0xc00040d860, 0x300000000000002, 0xffffffffffffffff, 0x0)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/maputil/maputil.go:27 +0x40f
github.com/roboll/helmfile/pkg/maputil.recursivelyStringifyMapKey(0x188d280, 0xc00040d860, 0xc0001eab88, 0xa, 0xc0000e6998, 0x0)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/maputil/maputil.go:42 +0x75
github.com/roboll/helmfile/pkg/maputil.CastKeysToStrings(0x188d280, 0xc00040d830, 0x30000c00033c6c0, 0xffffffffffffffff, 0x0)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/maputil/maputil.go:27 +0x40f
github.com/roboll/helmfile/pkg/maputil.recursivelyStringifyMapKey(0x188d280, 0xc00040d830, 0xc0001eadb0, 0xb, 0xc0000e6bc8, 0x0)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/maputil/maputil.go:42 +0x75
github.com/roboll/helmfile/pkg/maputil.CastKeysToStrings(0x188d280, 0xc00040d800, 0xc00033c660, 0xffffffffffffffff, 0x0)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/maputil/maputil.go:27 +0x40f
github.com/roboll/helmfile/pkg/maputil.recursivelyStringifyMapKey(0x188d280, 0xc00040d800, 0xc0001eafd8, 0x9, 0xc0000e6778, 0x0)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/maputil/maputil.go:42 +0x75
github.com/roboll/helmfile/pkg/maputil.CastKeysToStrings(0x188d280, 0xc00040d7d0, 0xc00033c5d0, 0x0, 0x10145c4)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/maputil/maputil.go:27 +0x40f
github.com/roboll/helmfile/pkg/maputil.recursivelyStringifyMapKey(0x188d280, 0xc00040d7d0, 0xc0000c97a0, 0xd, 0xc0000e62e8, 0x0)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/maputil/maputil.go:42 +0x75
github.com/roboll/helmfile/pkg/maputil.CastKeysToStrings(0x188d280, 0xc00033c270, 0x188d280, 0xc00040d740, 0xc0001eb320)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/maputil/maputil.go:27 +0x40f
github.com/roboll/helmfile/pkg/state.(*HelmState).Values(0xc0000c41e0, 0xc, 0xc0001eb398, 0x10e903a)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state_exec_tmpl.go:20 +0x169
github.com/roboll/helmfile/pkg/state.(*HelmState).mustLoadVals(...)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state_exec_tmpl.go:29
github.com/roboll/helmfile/pkg/state.(*HelmState).valuesFileTemplateData(0xc0000c41e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state_exec_tmpl.go:40 +0x51
github.com/roboll/helmfile/pkg/state.(*HelmState).RenderValuesFileToBytes(0xc0000c41e0, 0xc000414160, 0x18, 0x6, 0xc0004140a0, 0x18, 0xc0004900b0, 0x1)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state.go:1285 +0x72
github.com/roboll/helmfile/pkg/state.(*HelmState).generateTemporaryValuesFiles(0xc0000c41e0, 0xc0000ca1c0, 0x3, 0x4, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state.go:1342 +0x2ea
github.com/roboll/helmfile/pkg/state.(*HelmState).namespaceAndValuesFlags(0xc0000c41e0, 0x1b5ce60, 0xc00048e000, 0xc0001b0c00, 0x6, 0x21b2c88, 0x0, 0x0, 0x0, 0x0)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state.go:1394 +0x17a
github.com/roboll/helmfile/pkg/state.(*HelmState).flagsForDiff(0xc0000c41e0, 0x1b5ce60, 0xc00048e000, 0xc0001b0c00, 0x6, 0x0, 0x0, 0x0, 0x0, 0x0)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state.go:1254 +0x190
github.com/roboll/helmfile/pkg/state.(*HelmState).prepareDiffReleases.func2(0x6)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state.go:701 +0x5a0
github.com/roboll/helmfile/pkg/state.(*HelmState).scatterGather.func1(0xc0000c41e0, 0xc000036e38, 0xc0000e0fa0, 0xc000036e60, 0x6)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state_run.go:41 +0x125
created by github.com/roboll/helmfile/pkg/state.(*HelmState).scatterGather
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state_run.go:39 +0x1bb

goroutine 87 [running]:
runtime.throw(0x19b864b, 0x15)
	/usr/local/Cellar/go/1.12.5/libexec/src/runtime/panic.go:617 +0x72 fp=0xc0001a0b90 sp=0xc0001a0b60 pc=0x102db02
runtime.mapassign(0x188d280, 0xc00040dfb0, 0xc000156230, 0xc0001a0c50)
	/usr/local/Cellar/go/1.12.5/libexec/src/runtime/map.go:682 +0x5c8 fp=0xc0001a0c18 sp=0xc0001a0b90 pc=0x100fd48
reflect.mapassign(0x188d280, 0xc00040dfb0, 0xc000156230, 0xc000156280)
	/usr/local/Cellar/go/1.12.5/libexec/src/runtime/map.go:1331 +0x3f fp=0xc0001a0c48 sp=0xc0001a0c18 pc=0x10117ef
reflect.Value.SetMapIndex(0x188d280, 0xc00040dfb0, 0x15, 0x1844e40, 0xc000156230, 0x98, 0x1888fc0, 0xc000156280, 0x94)
	/usr/local/Cellar/go/1.12.5/libexec/src/reflect/value.go:1615 +0x234 fp=0xc0001a0cb0 sp=0xc0001a0c48 pc=0x1088b04
github.com/imdario/mergo.deepMerge(0x188d280, 0xc00040dfb0, 0x15, 0x188d280, 0xc00040d7a0, 0x15, 0xc0001a1200, 0x1, 0xc00050c080, 0x0, ...)
	/Users/dog/go/pkg/mod/github.com/imdario/mergo@v0.3.6/merge.go:147 +0x677 fp=0xc0001a0f00 sp=0xc0001a0cb0 pc=0x171f947
github.com/imdario/mergo.deepMerge(0x188d280, 0xc0001d0010, 0x195, 0x188d280, 0xc00040d740, 0x15, 0xc0001a1200, 0x0, 0xc00050c080, 0x15, ...)
	/Users/dog/go/pkg/mod/github.com/imdario/mergo@v0.3.6/merge.go:115 +0x97d fp=0xc0001a1150 sp=0xc0001a0f00 pc=0x171fc4d
github.com/imdario/mergo.merge(0x18274a0, 0xc0001d0010, 0x188d280, 0xc00040d740, 0xc0001a1320, 0x1, 0x1, 0x0, 0x0)
	/Users/dog/go/pkg/mod/github.com/imdario/mergo@v0.3.6/merge.go:251 +0x299 fp=0xc0001a12d0 sp=0xc0001a1150 pc=0x1721d89
github.com/imdario/mergo.Merge(...)
	/Users/dog/go/pkg/mod/github.com/imdario/mergo@v0.3.6/merge.go:206
github.com/roboll/helmfile/pkg/state.(*HelmState).Values(0xc0000c41e0, 0xc, 0xc0001a1398, 0x10e903a)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state_exec_tmpl.go:16 +0x139 fp=0xc0001a1340 sp=0xc0001a12d0 pc=0x175e659
github.com/roboll/helmfile/pkg/state.(*HelmState).mustLoadVals(...)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state_exec_tmpl.go:29
github.com/roboll/helmfile/pkg/state.(*HelmState).valuesFileTemplateData(0xc0000c41e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state_exec_tmpl.go:40 +0x51 fp=0xc0001a13a8 sp=0xc0001a1340 pc=0x175e7c1
github.com/roboll/helmfile/pkg/state.(*HelmState).RenderValuesFileToBytes(0xc0000c41e0, 0xc000438280, 0x18, 0x6, 0xc0004381c0, 0x18, 0xc000156080, 0x1)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state.go:1285 +0x72 fp=0xc0001a1478 sp=0xc0001a13a8 pc=0x1759e92
github.com/roboll/helmfile/pkg/state.(*HelmState).generateTemporaryValuesFiles(0xc0000c41e0, 0xc000476080, 0x3, 0x4, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state.go:1342 +0x2ea fp=0xc0001a1898 sp=0xc0001a1478 pc=0x175a9ea
github.com/roboll/helmfile/pkg/state.(*HelmState).namespaceAndValuesFlags(0xc0000c41e0, 0x1b5ce60, 0xc00048e000, 0xc0001b13a0, 0x5, 0x21b2c88, 0x0, 0x0, 0x0, 0x0)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state.go:1394 +0x17a fp=0xc0001a1c80 sp=0xc0001a1898 pc=0x175b91a
github.com/roboll/helmfile/pkg/state.(*HelmState).flagsForDiff(0xc0000c41e0, 0x1b5ce60, 0xc00048e000, 0xc0001b13a0, 0x5, 0x0, 0x0, 0x0, 0x0, 0x0)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state.go:1254 +0x190 fp=0xc0001a1d28 sp=0xc0001a1c80 pc=0x17599b0
github.com/roboll/helmfile/pkg/state.(*HelmState).prepareDiffReleases.func2(0x5)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state.go:701 +0x5a0 fp=0xc0001a1f00 sp=0xc0001a1d28 pc=0x1763a40
github.com/roboll/helmfile/pkg/state.(*HelmState).scatterGather.func1(0xc0000c41e0, 0xc000036e38, 0xc0000e0fa0, 0xc000036e60, 0x5)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state_run.go:41 +0x125 fp=0xc0001a1fb8 sp=0xc0001a1f00 pc=0x1765b75
runtime.goexit()
	/usr/local/Cellar/go/1.12.5/libexec/src/runtime/asm_amd64.s:1337 +0x1 fp=0xc0001a1fc0 sp=0xc0001a1fb8 pc=0x105a7b1
created by github.com/roboll/helmfile/pkg/state.(*HelmState).scatterGather
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state_run.go:39 +0x1bb

goroutine 84 [running]:
runtime.throw(0x19b864b, 0x15)
	/usr/local/Cellar/go/1.12.5/libexec/src/runtime/panic.go:617 +0x72 fp=0xc00019eb90 sp=0xc00019eb60 pc=0x102db02
runtime.mapassign(0x188d280, 0xc00040dfb0, 0xc0001d9e60, 0xc00019ec50)
	/usr/local/Cellar/go/1.12.5/libexec/src/runtime/map.go:590 +0x5e3 fp=0xc00019ec18 sp=0xc00019eb90 pc=0x100fd63
reflect.mapassign(0x188d280, 0xc00040dfb0, 0xc0001d9e60, 0xc0001d9e80)
	/usr/local/Cellar/go/1.12.5/libexec/src/runtime/map.go:1331 +0x3f fp=0xc00019ec48 sp=0xc00019ec18 pc=0x10117ef
reflect.Value.SetMapIndex(0x188d280, 0xc00040dfb0, 0x15, 0x1844e40, 0xc0001d9e60, 0x98, 0x1888fc0, 0xc0001d9e80, 0x94)
	/usr/local/Cellar/go/1.12.5/libexec/src/reflect/value.go:1615 +0x234 fp=0xc00019ecb0 sp=0xc00019ec48 pc=0x1088b04
github.com/imdario/mergo.deepMerge(0x188d280, 0xc00040dfb0, 0x15, 0x188d280, 0xc00040d7a0, 0x15, 0xc00019f200, 0x1, 0xc0000c3b60, 0x0, ...)
	/Users/dog/go/pkg/mod/github.com/imdario/mergo@v0.3.6/merge.go:147 +0x677 fp=0xc00019ef00 sp=0xc00019ecb0 pc=0x171f947
github.com/imdario/mergo.deepMerge(0x188d280, 0xc000010100, 0x195, 0x188d280, 0xc00040d740, 0x15, 0xc00019f200, 0x0, 0xc0000c3b60, 0x15, ...)
	/Users/dog/go/pkg/mod/github.com/imdario/mergo@v0.3.6/merge.go:115 +0x97d fp=0xc00019f150 sp=0xc00019ef00 pc=0x171fc4d
github.com/imdario/mergo.merge(0x18274a0, 0xc000010100, 0x188d280, 0xc00040d740, 0xc00019f320, 0x1, 0x1, 0x0, 0x0)
	/Users/dog/go/pkg/mod/github.com/imdario/mergo@v0.3.6/merge.go:251 +0x299 fp=0xc00019f2d0 sp=0xc00019f150 pc=0x1721d89
github.com/imdario/mergo.Merge(...)
	/Users/dog/go/pkg/mod/github.com/imdario/mergo@v0.3.6/merge.go:206
github.com/roboll/helmfile/pkg/state.(*HelmState).Values(0xc0000c41e0, 0xc, 0xc00019f398, 0x10e903a)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state_exec_tmpl.go:16 +0x139 fp=0xc00019f340 sp=0xc00019f2d0 pc=0x175e659
github.com/roboll/helmfile/pkg/state.(*HelmState).mustLoadVals(...)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state_exec_tmpl.go:29
github.com/roboll/helmfile/pkg/state.(*HelmState).valuesFileTemplateData(0xc0000c41e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state_exec_tmpl.go:40 +0x51 fp=0xc00019f3a8 sp=0xc00019f340 pc=0x175e7c1
github.com/roboll/helmfile/pkg/state.(*HelmState).RenderValuesFileToBytes(0xc0000c41e0, 0xc000438140, 0x18, 0x6, 0xc000438080, 0x18, 0xc0001d9d90, 0x1)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state.go:1285 +0x72 fp=0xc00019f478 sp=0xc00019f3a8 pc=0x1759e92
github.com/roboll/helmfile/pkg/state.(*HelmState).generateTemporaryValuesFiles(0xc0000c41e0, 0xc000476040, 0x3, 0x4, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state.go:1342 +0x2ea fp=0xc00019f898 sp=0xc00019f478 pc=0x175a9ea
github.com/roboll/helmfile/pkg/state.(*HelmState).namespaceAndValuesFlags(0xc0000c41e0, 0x1b5ce60, 0xc00048e000, 0xc0001b0de8, 0x2, 0x21b2c88, 0x0, 0x0, 0x0, 0x0)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state.go:1394 +0x17a fp=0xc00019fc80 sp=0xc00019f898 pc=0x175b91a
github.com/roboll/helmfile/pkg/state.(*HelmState).flagsForDiff(0xc0000c41e0, 0x1b5ce60, 0xc00048e000, 0xc0001b0de8, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state.go:1254 +0x190 fp=0xc00019fd28 sp=0xc00019fc80 pc=0x17599b0
github.com/roboll/helmfile/pkg/state.(*HelmState).prepareDiffReleases.func2(0x2)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state.go:701 +0x5a0 fp=0xc00019ff00 sp=0xc00019fd28 pc=0x1763a40
github.com/roboll/helmfile/pkg/state.(*HelmState).scatterGather.func1(0xc0000c41e0, 0xc000036e38, 0xc0000e0fa0, 0xc000036e60, 0x2)
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state_run.go:41 +0x125 fp=0xc00019ffb8 sp=0xc00019ff00 pc=0x1765b75
runtime.goexit()
	/usr/local/Cellar/go/1.12.5/libexec/src/runtime/asm_amd64.s:1337 +0x1 fp=0xc00019ffc0 sp=0xc00019ffb8 pc=0x105a7b1
created by github.com/roboll/helmfile/pkg/state.(*HelmState).scatterGather
	/Users/dog/Documents/xyz/code/github.com/bitsofinfo/helmfile/pkg/state/state_run.go:39 +0x1bb