{"body":"# tail -f $(svcs -L manatee-sitter) | bunyan -o short -l debug\n19:49:10.554Z INFO manatee-sitter/PostgresMgr: PostgresMgr.restart: entering\n19:49:10.567Z DEBUG manatee-sitter/PostgresMgr: pg queue kicked: outstanding request already\n19:49:14.015Z DEBUG manatee-sitter/ZookeeperMgr: zk: handling active init/notification (localZk=null, selfZk=0360ff725a6e002b, res.version=0)\n res.data: [\n \"10.0.8.61:5432:12345-0000000132\",\n \"10.0.8.60:5432:12345-0000000131\",\n \"10.0.8.14:5432:12345-0000000135\"\n ]\n19:49:14.016Z DEBUG manatee-sitter/ZookeeperMgr: zk: end handling active init/notification (inited=true, hasCb=false, zk=0360ff725a6e002b)\n active: [\n {\n \"id\": \"10.0.8.14:5432:12345\",\n \"seq\": 135,\n \"name\": \"10.0.8.14:5432:12345-0000000135\",\n \"data\": {\n \"zoneId\": \"4ff8d0ee-4b07-4cac-954e-38b92470a907\",\n \"ip\": \"10.0.8.14\",\n \"pgUrl\": \"tcp://postgres@10.0.8.14:5432/postgres\",\n \"backupUrl\": \"http://10.0.8.14:12345\"\n }\n },\n {\n \"id\": \"10.0.8.60:5432:12345\",\n \"seq\": 131,\n \"name\": \"10.0.8.60:5432:12345-0000000131\",\n \"data\": {\n \"zoneId\": \"1ae54ead-73c6-424f-82aa-9c837d046cdf\",\n \"ip\": \"10.0.8.60\",\n \"pgUrl\": \"tcp://postgres@10.0.8.60:5432/postgres\",\n \"backupUrl\": \"http://10.0.8.60:12345\"\n }\n },\n {\n \"id\": \"10.0.8.61:5432:12345\",\n \"seq\": 132,\n \"name\": \"10.0.8.61:5432:12345-0000000132\",\n \"data\": {\n \"zoneId\": \"e3292f47-59f5-4f6b-8535-062220077ab7\",\n \"ip\": \"10.0.8.61\",\n \"pgUrl\": \"tcp://postgres@10.0.8.61:5432/postgres\",\n \"backupUrl\": \"http://10.0.8.61:12345\"\n }\n }\n ]\n19:49:20.672Z DEBUG manatee-sitter/PostgresMgr: pg queue kicked: outstanding request already\n19:49:27.496Z ERROR manatee-sitter/PostgresMgr: got pg client error (err.code=ECONNREFUSED)\n Error: connect ECONNREFUSED\n at errnoException (net.js:904:11)\n at Object.afterConnect [as oncomplete] (net.js:895:19)\n19:49:27.497Z WARN manatee-sitter/PostgresMgr: got err (err.code=ECONNREFUSED)\n Error: connect ECONNREFUSED\n at errnoException (net.js:904:11)\n at Object.afterConnect [as oncomplete] (net.js:895:19)\n19:49:27.499Z INFO manatee-sitter/PostgresMgr: PostgresMgr.restart: db not running\n19:49:27.499Z INFO manatee-sitter/PostgresMgr: PostgresMgr.start: entering (postgresPath=/opt/local/bin/postgres, dataDir=/manatee/pg/data)\n19:49:27.534Z ERROR manatee-sitter/PostgresMgr: got pg client error (err.code=ECONNREFUSED)\n Error: connect ECONNREFUSED\n at errnoException (net.js:904:11)\n at Object.afterConnect [as oncomplete] (net.js:895:19)\n19:49:27.534Z WARN manatee-sitter/PostgresMgr: got err (err.code=ECONNREFUSED)\n Error: connect ECONNREFUSED\n at errnoException (net.js:904:11)\n at Object.afterConnect [as oncomplete] (net.js:895:19)\n19:49:27.539Z ERROR manatee-sitter/PostgresMgr: got pg client error (err.code=ECONNREFUSED)\n Error: connect ECONNREFUSED\n at errnoException (net.js:904:11)\n at Object.afterConnect [as oncomplete] (net.js:895:19)\n19:49:27.539Z WARN manatee-sitter/PostgresMgr: got err (err.code=ECONNREFUSED)\n Error: connect ECONNREFUSED\n at errnoException (net.js:904:11)\n at Object.afterConnect [as oncomplete] (net.js:895:19)\n19:49:32.119Z DEBUG manatee-sitter/PostgresMgr: pg queue kicked: outstanding request already\n19:49:32.128Z ERROR manatee-sitter/PostgresMgr: got pg client error (err.code=ECONNRESET)\n Error: read ECONNRESET\n at errnoException (net.js:904:11)\n at TCP.onread (net.js:558:19)\n19:49:32.128Z WARN manatee-sitter/PostgresMgr: got err (err.code=ECONNRESET)\n Error: read ECONNRESET\n at errnoException (net.js:904:11)\n at TCP.onread (net.js:558:19)\n19:49:32.130Z INFO manatee-sitter/PostgresMgr: PostgresMgr.start: db has not started (timeSinceStart=4599, opsTimeout=300000, postgresPath=/opt/local/bin/postgres, dataDir=/manatee/pg/data)\n VError: error whilst querying postgres: read ECONNRESET\n at PostgresMgr._pgQueryFini (/opt/smartdc/manatee/node_modules/manatee/lib/postgresMgr.js:1685:15)\n at null. (/opt/smartdc/manatee/node_modules/manatee/lib/postgresMgr.js:1636:22)\n at g (events.js:180:16)\n at EventEmitter.emit (events.js:95:17)\n at null. (/opt/smartdc/manatee/node_modules/manatee/node_modules/pg/lib/client.js:170:19)\n at EventEmitter.emit (events.js:95:17)\n at Socket. (/opt/smartdc/manatee/node_modules/manatee/node_modules/pg/lib/connection.js:59:10)\n at Socket.EventEmitter.emit (events.js:95:17)\n at net.js:440:14\n at process._tickDomainCallback (node.js:459:13)\n Caused by: Error: read ECONNRESET\n at errnoException (net.js:904:11)\n at TCP.onread (net.js:558:19)\n19:49:32.132Z INFO manatee-sitter/PostgresMgr: PostgresMgr.start: postgres -D exited with err (postgresPath=/opt/local/bin/postgres, dataDir=/manatee/pg/data, code=1, signal=null)\n VError: postgres exited unexpectedly (code 1); stdout = , stderr = 2018-01-16 19:49:31.832 UTCFATAL: shmat(id=67108874) failed: Not enough space\n\n at ChildProcess. (/opt/smartdc/manatee/node_modules/manatee/lib/postgresMgr.js:1358:19)\n at ChildProcess.EventEmitter.emit (events.js:98:17)\n at Process.ChildProcess._handle.onexit (child_process.js:797:12)\n19:49:32.132Z INFO manatee-sitter/PostgresMgr: clearing healthcheck\n19:49:32.133Z ERROR manatee-sitter/PostgresMgr: PostgresMgr._primary: error (standby=tcp://postgres@10.0.8.61:5432/postgres)\n VError: postgres exited unexpectedly (code 1); stdout = , stderr = 2018-01-16 19:49:31.832 UTCFATAL: shmat(id=67108874) failed: Not enough space\n\n at ChildProcess. (/opt/smartdc/manatee/node_modules/manatee/lib/postgresMgr.js:1358:19)\n at ChildProcess.EventEmitter.emit (events.js:98:17)\n at Process.ChildProcess._handle.onexit (child_process.js:797:12)\n --\n url: {\n \"protocol\": \"tcp:\",\n \"slashes\": true,\n \"auth\": \"postgres\",\n \"host\": \"10.0.8.14:5432\",\n \"port\": \"5432\",\n \"hostname\": \"10.0.8.14\",\n \"hash\": null,\n \"search\": null,\n \"query\": null,\n \"pathname\": \"/postgres\",\n \"path\": \"/postgres\",\n \"href\": \"tcp://postgres@10.0.8.14:5432/postgres\"\n }\n19:49:32.133Z INFO manatee-sitter/PostgresMgr: PostgresMgr.start: exiting\n VError: postgres exited unexpectedly (code 1); stdout = , stderr = 2018-01-16 19:49:31.832 UTCFATAL: shmat(id=67108874) failed: Not enough space\n\n at ChildProcess. (/opt/smartdc/manatee/node_modules/manatee/lib/postgresMgr.js:1358:19)\n at ChildProcess.EventEmitter.emit (events.js:98:17)\n at Process.ChildProcess._handle.onexit (child_process.js:797:12)\n19:49:32.133Z ERROR manatee-sitter/cluster:\n applying pg config: postgres exited unexpectedly (code 1); stdout = , stderr = 2018-01-16 19:49:31.832 UTCFATAL: shmat(id=67108874) failed: Not enough space\n\n --\n VError: applying pg config: postgres exited unexpectedly (code 1); stdout = , stderr = 2018-01-16 19:49:31.832 UTCFATAL: shmat(id=67108874) failed: Not enough space\n\n at finishPgApply (/opt/smartdc/manatee/node_modules/manatee/node_modules/manatee-state-machine/lib/manatee-peer.js:1087:10)\n at next (/opt/smartdc/manatee/node_modules/manatee/node_modules/manatee-state-machine/node_modules/vasync/lib/vasync.js:581:14)\n at /opt/smartdc/manatee/node_modules/manatee/lib/postgresMgr.js:359:17\n at onReconfigure (/opt/smartdc/manatee/node_modules/manatee/lib/postgresMgr.js:612:17)\n at /opt/smartdc/manatee/node_modules/manatee/lib/postgresMgr.js:818:17\n at next (/opt/smartdc/manatee/node_modules/manatee/node_modules/vasync/lib/vasync.js:201:4)\n at f (/opt/smartdc/manatee/node_modules/manatee/node_modules/once/once.js:16:25)\n at /opt/smartdc/manatee/node_modules/manatee/lib/postgresMgr.js:1323:9\n at f (/opt/smartdc/manatee/node_modules/manatee/node_modules/once/once.js:16:25)\n at ChildProcess. (/opt/smartdc/manatee/node_modules/manatee/lib/postgresMgr.js:1398:16)\n Caused by: VError: postgres exited unexpectedly (code 1); stdout = , stderr = 2018-01-16 19:49:31.832 UTCFATAL: shmat(id=67108874) failed: Not enough space\n\n at ChildProcess. (/opt/smartdc/manatee/node_modules/manatee/lib/postgresMgr.js:1358:19)\n at ChildProcess.EventEmitter.emit (events.js:98:17)\n at Process.ChildProcess._handle.onexit (child_process.js:797:12)\n19:49:32.134Z ERROR manatee-sitter/PostgresMgr: got pg client error (err.code=ECONNREFUSED)\n Error: connect ECONNREFUSED\n at errnoException (net.js:904:11)\n at Object.afterConnect [as oncomplete] (net.js:895:19)\n19:49:32.134Z WARN manatee-sitter/PostgresMgr: got err (err.code=ECONNREFUSED)\n Error: connect ECONNREFUSED\n at errnoException (net.js:904:11)\n at Object.afterConnect [as oncomplete] (net.js:895:19)\n19:49:33.136Z DEBUG manatee-sitter/cluster:\n pg.reconfigure { role: 'primary',\n upstream: null,\n downstream:\n { id: '10.0.8.61:5432:12345',\n zoneId: 'e3292f47-59f5-4f6b-8535-062220077ab7',\n ip: '10.0.8.61',\n pgUrl: 'tcp://postgres@10.0.8.61:5432/postgres',\n backupUrl: 'http://10.0.8.61:12345' } }\n19:49:33.137Z DEBUG manatee-sitter/cluster: pg: enabling\n19:49:33.137Z INFO manatee-sitter/PostgresMgr: PostgresMgr.start: entering\n19:49:33.137Z INFO manatee-sitter/PostgresMgr: PostgresMgr._primary: entering. (standby=tcp://postgres@10.0.8.61:5432/postgres)\n url: {\n \"protocol\": \"tcp:\",\n \"slashes\": true,\n \"auth\": \"postgres\",\n \"host\": \"10.0.8.14:5432\",\n \"port\": \"5432\",\n \"hostname\": \"10.0.8.14\",\n \"hash\": null,\n \"search\": null,\n \"query\": null,\n \"pathname\": \"/postgres\",\n \"path\": \"/postgres\",\n \"href\": \"tcp://postgres@10.0.8.14:5432/postgres\"\n }","name":"","extension":"txt","url":"https://www.irccloud.com/pastebin/kQ9LBMZx","modified":1516132628,"id":"kQ9LBMZx","size":10088,"lines":185,"own_paste":false,"theme":"","date":1516132628}