ONAP on Rackspace
- Michael O'Brien
Rackspace volumes are an issue - they don't work like EBS volumes
There are also issues specific to rackspace that saturate the CPU's with network/FS processes - causing CD server timeouts
INT-347: Rackspace VM perodic 2400% CPU - various wipefs processesClosed
root@oom:~# kubectl get pods --all-namespaces
NAMESPACE NAME READY STATUS RESTARTS AGE
cattle-system kubernetes-support-kubectl-shell-1-d7507c8a 0/2 Init:0/1 0 18h
kube-system kubernetes-dashboard-1319779976-lnvp0 1/1 Running 0 18h
kube-system tiller-deploy-3771619545-97cbz 1/1 Running 0 18h
onap-aaf aaf-1993711932-nmx38 0/1 Running 0 18h
onap-aaf aaf-cs-1310404376-8vmsl 1/1 Running 0 18h
onap-aai aai-resources-898583818-1tls8 1/2 CrashLoopBackOff 198 18h
onap-aai aai-service-3869033750-2cwrv 0/1 Init:0/1 106 18h
onap-aai aai-traversal-2843093465-zstcs 0/2 Init:0/1 106 18h
onap-aai data-router-2254557428-647zg 0/1 ContainerCreating 0 18h
onap-aai elasticsearch-622738319-ztp8j 1/1 Running 0 18h
onap-aai hbase-1949550546-mnm4m 1/1 Running 0 18h
onap-aai model-loader-service-1260517459-kn7st 2/2 Running 0 18h
onap-aai search-data-service-2243336541-sd0g0 2/2 Running 0 18h
onap-aai sparky-be-1292339831-3t62x 2/2 Running 0 18h
onap-appc appc-3003924683-gf04d 1/2 Running 0 18h
onap-appc appc-dbhost-2793739621-wrn1v 1/1 Running 0 18h
onap-appc appc-dgbuilder-3378670973-50pjb 0/1 Init:0/1 106 18h
onap-clamp clamp-2925721051-4x7sv 0/1 CrashLoopBackOff 200 18h
onap-clamp clamp-mariadb-1812977665-b3g83 1/1 Running 0 18h
onap-cli cli-2059237210-tpmft 1/1 Running 0 18h
onap-consul consul-agent-3312409084-q82ch 1/1 Running 187 18h
onap-consul consul-server-1173049560-1qdp4 0/1 CrashLoopBackOff 188 18h
onap-consul consul-server-1173049560-fndrc 0/1 CrashLoopBackOff 187 18h
onap-consul consul-server-1173049560-j8wkh 0/1 CrashLoopBackOff 187 18h
onap-kube2msb kube2msb-registrator-4293827076-20brw 0/1 CrashLoopBackOff 197 18h
onap-log elasticsearch-1942187295-70xlj 0/1 Init:0/1 0 18h
onap-log kibana-3372627750-j0hwk 1/1 Running 0 18h
onap-log logstash-1708188010-dls0c 1/1 Running 0 18h
onap-message-router dmaap-3126594942-vj8b8 0/1 Init:0/1 106 18h
onap-message-router global-kafka-666408702-ws148 0/1 CrashLoopBackOff 195 18h
onap-message-router zookeeper-624700062-1j37n 1/1 Running 0 18h
onap-msb msb-consul-3334785600-5k4z5 1/1 Running 0 18h
onap-msb msb-discovery-3147497472-186n4 1/1 Running 0 18h
onap-msb msb-eag-1740223167-cfs9w 1/1 Running 0 18h
onap-msb msb-iag-3674463196-qnn27 1/1 Running 0 18h
onap-mso mariadb-829081257-lpnfw 1/1 Running 0 18h
onap-mso mso-821928192-rj9r6 2/2 Running 0 18h
onap-multicloud framework-3383338639-sxbqn 1/1 Running 0 18h
onap-multicloud multicloud-ocata-17965183-r3chk 1/1 Running 0 18h
onap-multicloud multicloud-vio-2514983946-fn1w8 1/1 Running 0 18h
onap-multicloud multicloud-windriver-4246375214-td4z6 1/1 Running 0 18h
onap-policy brmsgw-2863338584-m0s99 0/1 Init:0/1 105 18h
onap-policy drools-51784799-vvrrv 0/2 Init:0/1 106 18h
onap-policy mariadb-468860085-7w95f 1/1 Running 0 18h
onap-policy nexus-3012320886-vj617 1/1 Running 0 18h
onap-policy pap-302549538-4dbt5 0/2 PodInitializing 0 18h
onap-policy pdp-2150832668-x1vlf 0/2 Init:0/1 106 18h
onap-portal portalapps-1170512133-zqbp4 0/2 Init:CrashLoopBackOff 193 18h
onap-portal portaldb-84684679-xc207 2/2 Running 0 18h
onap-portal portalwidgets-774425896-8hwls 1/1 Running 0 18h
onap-portal vnc-portal-3680188324-d6gdv 0/1 Init:0/5 106 18h
onap-robot robot-4251390084-rz4pw 1/1 Running 0 18h
onap-sdc sdc-be-754421819-kghss 0/2 Init:Error 105 18h
onap-sdc sdc-cs-2937804434-66m21 1/1 Running 0 18h
onap-sdc sdc-es-2514443912-jpjg8 1/1 Running 0 18h
onap-sdc sdc-fe-902103934-fl0kj 0/2 Init:0/1 106 18h
onap-sdc sdc-kb-281446026-v7s9h 1/1 Running 0 18h
onap-sdnc sdnc-694965092-hg741 1/2 Running 0 18h
onap-sdnc sdnc-dbhost-3029711096-1zv4t 1/1 Running 0 18h
onap-sdnc sdnc-dgbuilder-356329770-cl9l5 0/1 Init:0/1 106 18h
onap-sdnc sdnc-portal-3065395015-c9v79 0/1 Init:Error 105 18h
onap-uui uui-4073331301-cf4d1 1/1 Running 0 18h
onap-vfc vfc-catalog-70337695-r1srv 0/1 Running 0 18h
onap-vfc vfc-emsdriver-1391521317-0xg9l 1/1 Running 0 18h
onap-vfc vfc-gvnfmdriver-483393592-3wqrg 1/1 Running 0 18h
onap-vfc vfc-hwvnfmdriver-2004241844-jkjqr 1/1 Running 0 18h
onap-vfc vfc-jujudriver-3538568144-f2vkw 1/1 Running 0 18h
onap-vfc vfc-nslcm-2146990483-cqbkm 0/1 Running 0 18h
onap-vfc vfc-resmgr-1584840697-pcscd 1/1 Running 0 18h
onap-vfc vfc-vnflcm-1404595268-mjz9t 0/1 Running 0 18h
onap-vfc vfc-vnfmgr-2009987442-wq5r3 0/1 Running 0 18h
onap-vfc vfc-vnfres-2038072720-j0p6h 0/1 Running 0 18h
onap-vfc vfc-workflow-2530549902-6x8w9 0/1 CrashLoopBackOff 215 18h
onap-vfc vfc-ztesdncdriver-3874954327-6ndc0 1/1 Running 0 18h
onap-vfc vfc-ztevmanagerdriver-1829785462-f5cvg 1/1 Running 0 18h
onap-vid vid-mariadb-3318685446-fw7c8 1/1 Running 0 18h
onap-vid vid-server-3816226769-f8x9m 2/2 Running 0 18h
onap-vnfsdk postgres-436836560-rmtb3 1/1 Running 0 18h
onap-vnfsdk refrepo-1924147637-bd2h9 1/1 Running 0 18h
root@oom:~# kubectl -n onap-sdnc describe sdnc-dbhost-3029711096-1zv4t
the server doesn't have a resource type "sdnc-dbhost-3029711096-1zv4t"
root@oom:~# kubectl -n onap-sdnc describe pod sdnc-dbhost-3029711096-1zv4t
Name: sdnc-dbhost-3029711096-1zv4t
Namespace: onap-sdnc
Node: oom/10.223.80.80
Start Time: Fri, 10 Nov 2017 03:25:39 +0000
Labels: app=sdnc-dbhost
pod-template-hash=3029711096
Status: Running
IP: 10.42.92.38
Created By: ReplicaSet/sdnc-dbhost-3029711096
Controlled By: ReplicaSet/sdnc-dbhost-3029711096
Containers:
sdnc-db-container:
Container ID: docker://779f99f179d870f6c216b4ecdc51a13bdf996b28f267d8d743b310322da26568
Image: mysql/mysql-server:5.6
Port: 3306/TCP
State: Running
Started: Fri, 10 Nov 2017 03:26:02 +0000
Ready: True
Restart Count: 0
Readiness: tcp-socket :3306 delay=5s timeout=1s period=10s #success=1 #failure=3
Environment:
MYSQL_ROOT_PASSWORD: openECOMP1.0
MYSQL_ROOT_HOST: %
Mounts:
/etc/localtime from localtime (ro)
/var/lib/mysql from sdnc-data (rw)
/var/run/secrets/kubernetes.io/serviceaccount from default-token-h4lz5 (ro)
Conditions:
Type Status
Initialized True
Ready True
PodScheduled True
Volumes:
localtime:
Type: HostPath (bare host directory volume)
Path: /etc/localtime
sdnc-data:
Type: PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
ClaimName: sdnc-db
ReadOnly: false
default-token-h4lz5:
Type: Secret (a volume populated by a Secret)
SecretName: default-token-h4lz5
Optional: false
QoS Class: BestEffort
Node-Selectors: <none>
Tolerations: node.alpha.kubernetes.io/notReady:NoExecute for 300s
node.alpha.kubernetes.io/unreachable:NoExecute for 300s
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Warning FailedScheduling 18h (x4 over 18h) default-scheduler PersistentVolumeClaim is not bound: "sdnc-db"
Normal Scheduled 18h default-scheduler Successfully assigned sdnc-dbhost-3029711096-1zv4t to oom
Normal SuccessfulMountVolume 18h kubelet, oom MountVolume.SetUp succeeded for volume "localtime"
Normal SuccessfulMountVolume 18h kubelet, oom MountVolume.SetUp succeeded for volume "onap-sdnc-db"
Normal SuccessfulMountVolume 18h kubelet, oom MountVolume.SetUp succeeded for volume "default-token-h4lz5"
Normal Pulling 18h kubelet, oom pulling image "mysql/mysql-server:5.6"
Normal Pulled 18h kubelet, oom Successfully pulled image "mysql/mysql-server:5.6"
Normal Created 18h kubelet, oom Created container
Normal Started 18h kubelet, oom Started container
Warning Unhealthy 18h kubelet, oom Readiness probe failed: dial tcp 10.42.92.38:3306: getsockopt: connection refused