# Pastebin kQ9LBMZx # tail -f $(svcs -L manatee-sitter) | bunyan -o short -l debug 19:49:10.554Z INFO manatee-sitter/PostgresMgr: PostgresMgr.restart: entering 19:49:10.567Z DEBUG manatee-sitter/PostgresMgr: pg queue kicked: outstanding request already 19:49:14.015Z DEBUG manatee-sitter/ZookeeperMgr: zk: handling active init/notification (localZk=null, selfZk=0360ff725a6e002b, res.version=0) res.data: [ "10.0.8.61:5432:12345-0000000132", "10.0.8.60:5432:12345-0000000131", "10.0.8.14:5432:12345-0000000135" ] 19:49:14.016Z DEBUG manatee-sitter/ZookeeperMgr: zk: end handling active init/notification (inited=true, hasCb=false, zk=0360ff725a6e002b) active: [ { "id": "10.0.8.14:5432:12345", "seq": 135, "name": "10.0.8.14:5432:12345-0000000135", "data": { "zoneId": "4ff8d0ee-4b07-4cac-954e-38b92470a907", "ip": "10.0.8.14", "pgUrl": "tcp://postgres@10.0.8.14:5432/postgres", "backupUrl": "http://10.0.8.14:12345" } }, { "id": "10.0.8.60:5432:12345", "seq": 131, "name": "10.0.8.60:5432:12345-0000000131", "data": { "zoneId": "1ae54ead-73c6-424f-82aa-9c837d046cdf", "ip": "10.0.8.60", "pgUrl": "tcp://postgres@10.0.8.60:5432/postgres", "backupUrl": "http://10.0.8.60:12345" } }, { "id": "10.0.8.61:5432:12345", "seq": 132, "name": "10.0.8.61:5432:12345-0000000132", "data": { "zoneId": "e3292f47-59f5-4f6b-8535-062220077ab7", "ip": "10.0.8.61", "pgUrl": "tcp://postgres@10.0.8.61:5432/postgres", "backupUrl": "http://10.0.8.61:12345" } } ] 19:49:20.672Z DEBUG manatee-sitter/PostgresMgr: pg queue kicked: outstanding request already 19:49:27.496Z ERROR manatee-sitter/PostgresMgr: got pg client error (err.code=ECONNREFUSED) Error: connect ECONNREFUSED at errnoException (net.js:904:11) at Object.afterConnect [as oncomplete] (net.js:895:19) 19:49:27.497Z WARN manatee-sitter/PostgresMgr: got err (err.code=ECONNREFUSED) Error: connect ECONNREFUSED at errnoException (net.js:904:11) at Object.afterConnect [as oncomplete] (net.js:895:19) 19:49:27.499Z INFO manatee-sitter/PostgresMgr: PostgresMgr.restart: db not running 19:49:27.499Z INFO manatee-sitter/PostgresMgr: PostgresMgr.start: entering (postgresPath=/opt/local/bin/postgres, dataDir=/manatee/pg/data) 19:49:27.534Z ERROR manatee-sitter/PostgresMgr: got pg client error (err.code=ECONNREFUSED) Error: connect ECONNREFUSED at errnoException (net.js:904:11) at Object.afterConnect [as oncomplete] (net.js:895:19) 19:49:27.534Z WARN manatee-sitter/PostgresMgr: got err (err.code=ECONNREFUSED) Error: connect ECONNREFUSED at errnoException (net.js:904:11) at Object.afterConnect [as oncomplete] (net.js:895:19) 19:49:27.539Z ERROR manatee-sitter/PostgresMgr: got pg client error (err.code=ECONNREFUSED) Error: connect ECONNREFUSED at errnoException (net.js:904:11) at Object.afterConnect [as oncomplete] (net.js:895:19) 19:49:27.539Z WARN manatee-sitter/PostgresMgr: got err (err.code=ECONNREFUSED) Error: connect ECONNREFUSED at errnoException (net.js:904:11) at Object.afterConnect [as oncomplete] (net.js:895:19) 19:49:32.119Z DEBUG manatee-sitter/PostgresMgr: pg queue kicked: outstanding request already 19:49:32.128Z ERROR manatee-sitter/PostgresMgr: got pg client error (err.code=ECONNRESET) Error: read ECONNRESET at errnoException (net.js:904:11) at TCP.onread (net.js:558:19) 19:49:32.128Z WARN manatee-sitter/PostgresMgr: got err (err.code=ECONNRESET) Error: read ECONNRESET at errnoException (net.js:904:11) at TCP.onread (net.js:558:19) 19:49:32.130Z INFO manatee-sitter/PostgresMgr: PostgresMgr.start: db has not started (timeSinceStart=4599, opsTimeout=300000, postgresPath=/opt/local/bin/postgres, dataDir=/manatee/pg/data) VError: error whilst querying postgres: read ECONNRESET at PostgresMgr._pgQueryFini (/opt/smartdc/manatee/node_modules/manatee/lib/postgresMgr.js:1685:15) at null. (/opt/smartdc/manatee/node_modules/manatee/lib/postgresMgr.js:1636:22) at g (events.js:180:16) at EventEmitter.emit (events.js:95:17) at null. (/opt/smartdc/manatee/node_modules/manatee/node_modules/pg/lib/client.js:170:19) at EventEmitter.emit (events.js:95:17) at Socket. (/opt/smartdc/manatee/node_modules/manatee/node_modules/pg/lib/connection.js:59:10) at Socket.EventEmitter.emit (events.js:95:17) at net.js:440:14 at process._tickDomainCallback (node.js:459:13) Caused by: Error: read ECONNRESET at errnoException (net.js:904:11) at TCP.onread (net.js:558:19) 19:49:32.132Z INFO manatee-sitter/PostgresMgr: PostgresMgr.start: postgres -D exited with err (postgresPath=/opt/local/bin/postgres, dataDir=/manatee/pg/data, code=1, signal=null) VError: postgres exited unexpectedly (code 1); stdout = , stderr = 2018-01-16 19:49:31.832 UTCFATAL: shmat(id=67108874) failed: Not enough space at ChildProcess. (/opt/smartdc/manatee/node_modules/manatee/lib/postgresMgr.js:1358:19) at ChildProcess.EventEmitter.emit (events.js:98:17) at Process.ChildProcess._handle.onexit (child_process.js:797:12) 19:49:32.132Z INFO manatee-sitter/PostgresMgr: clearing healthcheck 19:49:32.133Z ERROR manatee-sitter/PostgresMgr: PostgresMgr._primary: error (standby=tcp://postgres@10.0.8.61:5432/postgres) VError: postgres exited unexpectedly (code 1); stdout = , stderr = 2018-01-16 19:49:31.832 UTCFATAL: shmat(id=67108874) failed: Not enough space at ChildProcess. (/opt/smartdc/manatee/node_modules/manatee/lib/postgresMgr.js:1358:19) at ChildProcess.EventEmitter.emit (events.js:98:17) at Process.ChildProcess._handle.onexit (child_process.js:797:12) -- url: { "protocol": "tcp:", "slashes": true, "auth": "postgres", "host": "10.0.8.14:5432", "port": "5432", "hostname": "10.0.8.14", "hash": null, "search": null, "query": null, "pathname": "/postgres", "path": "/postgres", "href": "tcp://postgres@10.0.8.14:5432/postgres" } 19:49:32.133Z INFO manatee-sitter/PostgresMgr: PostgresMgr.start: exiting VError: postgres exited unexpectedly (code 1); stdout = , stderr = 2018-01-16 19:49:31.832 UTCFATAL: shmat(id=67108874) failed: Not enough space at ChildProcess. (/opt/smartdc/manatee/node_modules/manatee/lib/postgresMgr.js:1358:19) at ChildProcess.EventEmitter.emit (events.js:98:17) at Process.ChildProcess._handle.onexit (child_process.js:797:12) 19:49:32.133Z ERROR manatee-sitter/cluster: applying pg config: postgres exited unexpectedly (code 1); stdout = , stderr = 2018-01-16 19:49:31.832 UTCFATAL: shmat(id=67108874) failed: Not enough space -- VError: applying pg config: postgres exited unexpectedly (code 1); stdout = , stderr = 2018-01-16 19:49:31.832 UTCFATAL: shmat(id=67108874) failed: Not enough space at finishPgApply (/opt/smartdc/manatee/node_modules/manatee/node_modules/manatee-state-machine/lib/manatee-peer.js:1087:10) at next (/opt/smartdc/manatee/node_modules/manatee/node_modules/manatee-state-machine/node_modules/vasync/lib/vasync.js:581:14) at /opt/smartdc/manatee/node_modules/manatee/lib/postgresMgr.js:359:17 at onReconfigure (/opt/smartdc/manatee/node_modules/manatee/lib/postgresMgr.js:612:17) at /opt/smartdc/manatee/node_modules/manatee/lib/postgresMgr.js:818:17 at next (/opt/smartdc/manatee/node_modules/manatee/node_modules/vasync/lib/vasync.js:201:4) at f (/opt/smartdc/manatee/node_modules/manatee/node_modules/once/once.js:16:25) at /opt/smartdc/manatee/node_modules/manatee/lib/postgresMgr.js:1323:9 at f (/opt/smartdc/manatee/node_modules/manatee/node_modules/once/once.js:16:25) at ChildProcess. (/opt/smartdc/manatee/node_modules/manatee/lib/postgresMgr.js:1398:16) Caused by: VError: postgres exited unexpectedly (code 1); stdout = , stderr = 2018-01-16 19:49:31.832 UTCFATAL: shmat(id=67108874) failed: Not enough space at ChildProcess. (/opt/smartdc/manatee/node_modules/manatee/lib/postgresMgr.js:1358:19) at ChildProcess.EventEmitter.emit (events.js:98:17) at Process.ChildProcess._handle.onexit (child_process.js:797:12) 19:49:32.134Z ERROR manatee-sitter/PostgresMgr: got pg client error (err.code=ECONNREFUSED) Error: connect ECONNREFUSED at errnoException (net.js:904:11) at Object.afterConnect [as oncomplete] (net.js:895:19) 19:49:32.134Z WARN manatee-sitter/PostgresMgr: got err (err.code=ECONNREFUSED) Error: connect ECONNREFUSED at errnoException (net.js:904:11) at Object.afterConnect [as oncomplete] (net.js:895:19) 19:49:33.136Z DEBUG manatee-sitter/cluster: pg.reconfigure { role: 'primary', upstream: null, downstream: { id: '10.0.8.61:5432:12345', zoneId: 'e3292f47-59f5-4f6b-8535-062220077ab7', ip: '10.0.8.61', pgUrl: 'tcp://postgres@10.0.8.61:5432/postgres', backupUrl: 'http://10.0.8.61:12345' } } 19:49:33.137Z DEBUG manatee-sitter/cluster: pg: enabling 19:49:33.137Z INFO manatee-sitter/PostgresMgr: PostgresMgr.start: entering 19:49:33.137Z INFO manatee-sitter/PostgresMgr: PostgresMgr._primary: entering. (standby=tcp://postgres@10.0.8.61:5432/postgres) url: { "protocol": "tcp:", "slashes": true, "auth": "postgres", "host": "10.0.8.14:5432", "port": "5432", "hostname": "10.0.8.14", "hash": null, "search": null, "query": null, "pathname": "/postgres", "path": "/postgres", "href": "tcp://postgres@10.0.8.14:5432/postgres" }