Cloud Pak for Data

 View Only
Expand all | Collapse all

db2 instance etcd pod restarts

  • 1.  db2 instance etcd pod restarts

    Posted Mon March 20, 2023 09:19 AM
    Edited by Andrey Ilinskiy Mon March 20, 2023 09:25 AM

    We have a clean installation of CPD 4.5.0 on OCP 4.10.38

    Created successfully db2 instance, but every certain minute etcd service for it restarts.

    Any ideas why it can happens? 

    Here is log from the pod

    + '[' -z 1 ']'
    + '[' -z c-db2oltp-1679304117990218-etcd ']'
    + NUM_RETRIES=300
    + iter=1
    + '[' '!' -d /persistence ']'
    + '[' 1 -gt 300 ']'
    ++ hostname
    + HOSTNAME=c-db2oltp-1679304117990218-etcd-0
    + '[' '!' -d /persistence/etcd/c-db2oltp-1679304117990218-etcd-0 ']'
    + ln -sf /persistence/etcd/c-db2oltp-1679304117990218-etcd-0 /var/run/etcd
    Re-joining etcd member
    + '[' -e /var/run/etcd/default.etcd ']'
    + echo 'Re-joining etcd member'
    + '[' '!' -f /var/run/etcd/member_id ']'
    ++ cat /var/run/etcd/member_id
    + member_id=dc7c4d39e57e8d04
    + true
    ++ eps
    ++ EPS=
    +++ seq 0 0
    ++ for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1)))
    ++ EPS=http://c-db2oltp-1679304117990218-etcd-0.c-db2oltp-1679304117990218-etcd:2379
    ++ echo http://c-db2oltp-1679304117990218-etcd-0.c-db2oltp-1679304117990218-etcd:2379
    + ETCDCTL_ENDPOINT=http://c-db2oltp-1679304117990218-etcd-0.c-db2oltp-1679304117990218-etcd:2379
    + etcdctl member update dc7c4d39e57e8d04 http://c-db2oltp-1679304117990218-etcd-0.c-db2oltp-1679304117990218-etcd:2380
    Error: client: etcd cluster is unavailable or misconfigured; error #0: dial tcp 10.128.2.105:2379: connect: connection refused
    error #0: dial tcp 10.128.2.105:2379: connect: connection refused
    + exec etcd --name c-db2oltp-1679304117990218-etcd-0 --listen-peer-urls http://0.0.0.0:2380 --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://c-db2oltp-1679304117990218-etcd-0.c-db2oltp-1679304117990218-etcd:2379 --data-dir /var/run/etcd/default.etcd --enable-v2 --logger=zap
    {"level":"info","ts":"2023-03-20T12:14:22.123Z","caller":"etcdmain/etcd.go:100","msg":"detected default host for advertise","host":"10.128.2.105"}
    {"level":"info","ts":"2023-03-20T12:14:22.124Z","caller":"etcdmain/etcd.go:134","msg":"server has been already initialized","data-dir":"/var/run/etcd/default.etcd","dir-type":"member"}
    {"level":"info","ts":"2023-03-20T12:14:22.124Z","caller":"embed/etcd.go:117","msg":"configuring peer listeners","listen-peer-urls":["http://0.0.0.0:2380"]}
    {"level":"info","ts":"2023-03-20T12:14:22.124Z","caller":"embed/etcd.go:127","msg":"configuring client listeners","listen-client-urls":["http://0.0.0.0:2379"]}
    {"level":"info","ts":"2023-03-20T12:14:22.125Z","caller":"embed/etcd.go:302","msg":"starting an etcd server","etcd-version":"3.4.14","git-sha":"8a03d2e96","go-version":"go1.12.17","go-os":"linux","go-arch":"amd64","max-cpu-set":12,"max-cpu-available":12,"member-initialized":true,"name":"c-db2oltp-1679304117990218-etcd-0","data-dir":"/var/run/etcd/default.etcd","wal-dir":"","wal-dir-dedicated":"","member-dir":"/var/run/etcd/default.etcd/member","force-new-cluster":false,"heartbeat-interval":"100ms","election-timeout":"1s","initial-election-tick-advance":true,"snapshot-count":100000,"snapshot-catchup-entries":5000,"initial-advertise-peer-urls":["http://10.128.2.105:2380"],"listen-peer-urls":["http://0.0.0.0:2380"],"advertise-client-urls":["http://c-db2oltp-1679304117990218-etcd-0.c-db2oltp-1679304117990218-etcd:2379"],"listen-client-urls":["http://0.0.0.0:2379"],"listen-metrics-urls":[],"cors":["*"],"host-whitelist":["*"],"initial-cluster":"","initial-cluster-state":"new","initial-cluster-token":"","quota-si...
    {"level":"info","ts":"2023-03-20T12:14:22.232Z","caller":"etcdserver/backend.go:80","msg":"opened backend db","path":"/var/run/etcd/default.etcd/member/snap/db","took":"59.630471ms"}
    {"level":"info","ts":"2023-03-20T12:14:23.819Z","caller":"etcdserver/raft.go:536","msg":"restarting local member","cluster-id":"af02b41153db4232","local-member-id":"dc7c4d39e57e8d04","commit-index":34564}
    {"level":"info","ts":"2023-03-20T12:14:23.820Z","caller":"raft/raft.go:1530","msg":"dc7c4d39e57e8d04 switched to configuration voters=()"}
    {"level":"info","ts":"2023-03-20T12:14:23.820Z","caller":"raft/raft.go:700","msg":"dc7c4d39e57e8d04 became follower at term 4"}
    {"level":"info","ts":"2023-03-20T12:14:23.820Z","caller":"raft/raft.go:383","msg":"newRaft dc7c4d39e57e8d04 [peers: [], term: 4, commit: 34564, applied: 0, lastindex: 34564, lastterm: 4]"}
    {"level":"warn","ts":"2023-03-20T12:14:23.858Z","caller":"auth/store.go:1366","msg":"simple token is not cryptographically signed"}
    {"level":"info","ts":"2023-03-20T12:14:23.875Z","caller":"etcdserver/quota.go:98","msg":"enabled backend quota with default value","quota-name":"v3-applier","quota-size-bytes":2147483648,"quota-size":"2.1 GB"}
    {"level":"info","ts":"2023-03-20T12:14:23.900Z","caller":"etcdserver/server.go:803","msg":"starting etcd server","local-member-id":"dc7c4d39e57e8d04","local-server-version":"3.4.14","cluster-version":"to_be_decided"}
    {"level":"info","ts":"2023-03-20T12:14:23.901Z","caller":"etcdserver/server.go:691","msg":"starting initial election tick advance","election-ticks":10}
    {"level":"info","ts":"2023-03-20T12:14:23.902Z","caller":"raft/raft.go:1530","msg":"dc7c4d39e57e8d04 switched to configuration voters=(15887658496515017988)"}
    {"level":"info","ts":"2023-03-20T12:14:23.902Z","caller":"membership/cluster.go:392","msg":"added member","cluster-id":"af02b41153db4232","local-member-id":"dc7c4d39e57e8d04","added-peer-id":"dc7c4d39e57e8d04","added-peer-peer-urls":["http://c-db2oltp-1679304117990218-etcd-0.c-db2oltp-1679304117990218-etcd:2380"]}
    {"level":"info","ts":"2023-03-20T12:14:23.902Z","caller":"membership/cluster.go:558","msg":"set initial cluster version","cluster-id":"af02b41153db4232","local-member-id":"dc7c4d39e57e8d04","cluster-version":"3.4"}
    {"level":"info","ts":"2023-03-20T12:14:23.902Z","caller":"api/capability.go:76","msg":"enabled capabilities for version","cluster-version":"3.4"}
    {"level":"info","ts":"2023-03-20T12:14:23.902Z","caller":"embed/etcd.go:244","msg":"now serving peer/client/metrics","local-member-id":"dc7c4d39e57e8d04","initial-advertise-peer-urls":["http://10.128.2.105:2380"],"listen-peer-urls":["http://0.0.0.0:2380"],"advertise-client-urls":["http://c-db2oltp-1679304117990218-etcd-0.c-db2oltp-1679304117990218-etcd:2379"],"listen-client-urls":["http://0.0.0.0:2379"],"listen-metrics-urls":[]}
    {"level":"info","ts":"2023-03-20T12:14:23.902Z","caller":"embed/etcd.go:579","msg":"serving peer traffic","address":"[::]:2380"}
    {"level":"info","ts":"2023-03-20T12:14:24.952Z","caller":"raft/raft.go:923","msg":"dc7c4d39e57e8d04 is starting a new election at term 4"}
    {"level":"info","ts":"2023-03-20T12:14:24.952Z","caller":"raft/raft.go:713","msg":"dc7c4d39e57e8d04 became candidate at term 5"}
    {"level":"info","ts":"2023-03-20T12:14:24.952Z","caller":"raft/raft.go:824","msg":"dc7c4d39e57e8d04 received MsgVoteResp from dc7c4d39e57e8d04 at term 5"}
    {"level":"info","ts":"2023-03-20T12:14:24.952Z","caller":"raft/raft.go:765","msg":"dc7c4d39e57e8d04 became leader at term 5"}
    {"level":"info","ts":"2023-03-20T12:14:24.952Z","caller":"raft/node.go:325","msg":"raft.node: dc7c4d39e57e8d04 elected leader dc7c4d39e57e8d04 at term 5"}
    {"level":"info","ts":"2023-03-20T12:14:25.014Z","caller":"etcdserver/server.go:2037","msg":"published local member to cluster through raft","local-member-id":"dc7c4d39e57e8d04","local-member-attributes":"{Name:c-db2oltp-1679304117990218-etcd-0 ClientURLs:[http://c-db2oltp-1679304117990218-etcd-0.c-db2oltp-1679304117990218-etcd:2379]}","request-path":"/0/members/dc7c4d39e57e8d04/attributes","cluster-id":"af02b41153db4232","publish-timeout":"7s"}
    {"level":"info","ts":"2023-03-20T12:14:25.015Z","caller":"embed/serve.go:139","msg":"serving client traffic insecurely; this is strongly discouraged!","address":"[::]:2379"}
    {"level":"warn","ts":"2023-03-20T12:18:16.442Z","caller":"wal/wal.go:806","msg":"slow fdatasync","took":"2.090545344s","expected-duration":"1s"}
    {"level":"warn","ts":"2023-03-20T12:18:17.583Z","caller":"wal/wal.go:806","msg":"slow fdatasync","took":"1.140651373s","expected-duration":"1s"}
    {"level":"warn","ts":"2023-03-20T12:18:18.868Z","caller":"wal/wal.go:806","msg":"slow fdatasync","took":"1.284531535s","expected-duration":"1s"}
    {"level":"warn","ts":"2023-03-20T12:18:20.638Z","caller":"wal/wal.go:806","msg":"slow fdatasync","took":"1.77040639s","expected-duration":"1s"}
    {"level":"warn","ts":"2023-03-20T12:18:22.694Z","caller":"wal/wal.go:806","msg":"slow fdatasync","took":"1.842527437s","expected-duration":"1s"}
    {"level":"warn","ts":"2023-03-20T12:18:25.703Z","caller":"wal/wal.go:806","msg":"slow fdatasync","took":"2.109065518s","expected-duration":"1s"}
    {"level":"warn","ts":"2023-03-20T12:42:33.976Z","caller":"wal/wal.go:806","msg":"slow fdatasync","took":"1.624493452s","expected-duration":"1s"}
    {"level":"warn","ts":"2023-03-20T12:58:22.498Z","caller":"wal/wal.go:806","msg":"slow fdatasync","took":"1.146099781s","expected-duration":"1s"}
    {"level":"warn","ts":"2023-03-20T12:58:23.858Z","caller":"wal/wal.go:806","msg":"slow fdatasync","took":"1.006274195s","expected-duration":"1s"}
    {"level":"warn","ts":"2023-03-20T13:00:05.338Z","caller":"wal/wal.go:806","msg":"slow fdatasync","took":"1.986167171s","expected-duration":"1s"}
    {"level":"warn","ts":"2023-03-20T13:00:08.108Z","caller":"wal/wal.go:806","msg":"slow fdatasync","took":"2.769588653s","expected-duration":"1s"}
    {"level":"warn","ts":"2023-03-20T13:00:10.519Z","caller":"wal/wal.go:806","msg":"slow fdatasync","took":"2.167276178s","expected-duration":"1s"}
    {"level":"warn","ts":"2023-03-20T13:00:13.328Z","caller":"wal/wal.go:806","msg":"slow fdatasync","took":"1.091471457s","expected-duration":"1s"}
    {"level":"warn","ts":"2023-03-20T13:00:19.138Z","caller":"wal/wal.go:806","msg":"slow fdatasync","took":"1.286190426s","expected-duration":"1s"}
    {"level":"warn","ts":"2023-03-20T13:00:41.793Z","caller":"wal/wal.go:806","msg":"slow fdatasync","took":"1.070657886s","expected-duration":"1s"}
    {"level":"warn","ts":"2023-03-20T13:01:00.629Z","caller":"v2http/client.go:607","msg":"unexpected v2 response error","internal-server-error":"etcdserver: request timed out"}
    {"level":"warn","ts":"2023-03-20T13:02:30.317Z","caller":"v2http/client.go:607","msg":"unexpected v2 response error","internal-server-error":"etcdserver: request timed out"}
    {"level":"warn","ts":"2023-03-20T13:03:56.813Z","caller":"wal/wal.go:806","msg":"slow fdatasync","took":"3m3.960991219s","expected-duration":"1s"}
    {"level":"fatal","ts":"2023-03-20T13:03:56.813Z","caller":"etcdserver/raft.go:251","msg":"failed to save Raft hard state and entries","error":"input/output error","stacktrace":"go.etcd.io/etcd/etcdserver.(*raftNode).start.func1\n\t/tmp/etcd-release-3.4.14/etcd/release/etcd/etcdserver/raft.go:251"}



    ------------------------------
    Andrey Ilinskiy
    ------------------------------