1#!/bin/sh 2# SPDX-License-Identifier: GPL-2.0-or-later 3# Copyright (c) International Business Machines Corp., 2007 4# Copyright (c) Linux Test Project, 2016-2020 5# Author: Sivakumar Chinnaiah <Sivakumar.C@in.ibm.com> 6# 7# Test Basic functionality of numactl command. 8# Test #1: Verifies cpunodebind and membind 9# Test #2: Verifies preferred node bind for memory allocation 10# Test #3: Verifies memory interleave on all nodes 11# Test #4: Verifies physcpubind 12# Test #5: Verifies localalloc 13# Test #6: Verifies memhog 14# Test #7: Verifies numa_node_size api 15# Test #8: Verifies hugepage alloacted on specified node 16# Test #9: Verifies THP memory allocated on preferred node 17 18TST_CNT=9 19TST_SETUP=setup 20TST_TESTFUNC=test 21TST_NEEDS_TMPDIR=1 22TST_NEEDS_ROOT=1 23TST_NEEDS_CMDS="awk bc numactl numastat" 24 25# Awk the field matching the node value for numastat 26# $1 - Pid number 27# $2 - Node number 28get_node_index() 29{ 30 local pid=$1 31 local nid="Node $2" 32 echo $(numastat -p $pid | sed '3q;d' | awk -F '[[:space:]][[:space:]]+' \ 33 -v node="$nid" '{ for (i = 1; i <= NF; ++i) if($i==node) print i; exit }') 34} 35 36# Convert the value of given NUMA node from the `numastat -p` output, 37# multiply by size. 38# $1 - Pid number 39# $2 - Node number 40# $3 - Size for multiplication (e.g. 1024, $MB) 41get_mem_cur() 42{ 43 local pid=$1 44 local index=$(echo "$(get_node_index $pid $2)") 45 local size=$3 46 local numstat=$(numastat -p $pid |awk '/^Total/ {print $'$index'}') 47 48 if [ -z "$numstat" ]; then 49 echo 0 50 return 51 fi 52 53 echo $(echo "$numstat * $size" | bc) 54} 55 56check_for_support_numa() 57{ 58 local pid=$1 59 60 local state=$(awk '{print $3}' /proc/$pid/stat) 61 62 if [ $state = 'T' ]; then 63 return 0 64 fi 65 66 return 1 67} 68 69setup() 70{ 71 export MB=$((1024*1024)) 72 export PAGE_SIZE=$(tst_getconf PAGESIZE) 73 export HPAGE_SIZE=$(awk '/Hugepagesize:/ {print $2}' /proc/meminfo) 74 75 total_nodes=0 76 77 nodes_list=$(numactl --show | grep nodebind | cut -d ':' -f 2) 78 for node in $nodes_list; do 79 total_nodes=$((total_nodes+1)) 80 done 81 82 tst_res TINFO "The system contains $total_nodes nodes: $nodes_list" 83 84 if [ $total_nodes -le 1 ]; then 85 tst_brk TCONF "SUT does not support NUMA policy or not a NUMA machine" 86 fi 87} 88 89# Verification of memory allocated on a node 90test1() 91{ 92 local mem_curr 93 94 for node in $nodes_list; do 95 numactl --cpunodebind=$node --membind=$node support_numa alloc_1MB & 96 pid=$! 97 98 TST_RETRY_FUNC "check_for_support_numa $pid" 0 99 100 mem_curr=$(get_mem_cur $pid $node $MB) 101 if [ $(echo "$mem_curr < $MB" | bc) -eq 1 ]; then 102 tst_res TFAIL \ 103 "NUMA memory allocated in node$node is less than expected" 104 kill -CONT $pid >/dev/null 2>&1 105 return 106 fi 107 108 kill -CONT $pid >/dev/null 2>&1 109 done 110 111 tst_res TPASS "NUMA local node and memory affinity" 112} 113 114# Verification of memory allocated on preferred node 115test2() 116{ 117 local mem_curr 118 local cnt=1 119 120 for node in $nodes_list; do 121 122 if [ $cnt -eq $total_nodes ]; then #wrap up for last node 123 Preferred_node=$(echo $nodes_list | cut -d ' ' -f 1) 124 else 125 # always next node is preferred node 126 Preferred_node=$(echo $nodes_list | cut -d ' ' -f $((cnt+1))) 127 fi 128 129 numactl --cpunodebind=$node --preferred=$Preferred_node support_numa alloc_1MB & 130 pid=$! 131 132 TST_RETRY_FUNC "check_for_support_numa $pid" 0 133 134 mem_curr=$(get_mem_cur $pid $Preferred_node $MB) 135 if [ $(echo "$mem_curr < $MB" |bc ) -eq 1 ]; then 136 tst_res TFAIL \ 137 "NUMA memory allocated in node$Preferred_node is less than expected" 138 kill -CONT $pid >/dev/null 2>&1 139 return 140 fi 141 142 cnt=$((cnt+1)) 143 kill -CONT $pid >/dev/null 2>&1 144 done 145 146 tst_res TPASS "NUMA preferred node policy" 147} 148 149# Verification of memory interleaved on all nodes 150test3() 151{ 152 local mem_curr 153 # Memory will be allocated using round robin on nodes. 154 Exp_incr=$(echo "$MB / $total_nodes" |bc) 155 156 numactl --interleave=all support_numa alloc_1MB & 157 pid=$! 158 159 TST_RETRY_FUNC "check_for_support_numa $pid" 0 160 161 for node in $nodes_list; do 162 mem_curr=$(get_mem_cur $pid $node $MB) 163 164 if [ $(echo "$mem_curr < $Exp_incr" |bc ) -eq 1 ]; then 165 tst_res TFAIL \ 166 "NUMA interleave memory allocated in node$node is less than expected" 167 kill -CONT $pid >/dev/null 2>&1 168 return 169 fi 170 done 171 172 kill -CONT $pid >/dev/null 2>&1 173 tst_res TPASS "NUMA interleave policy" 174} 175 176# Verification of physical cpu bind 177test4() 178{ 179 no_of_cpus=0 #no. of cpu's exist 180 run_on_cpu=0 181 running_on_cpu=0 182 183 no_of_cpus=$(tst_ncpus) 184 # not sure whether cpu's can't be in odd number 185 run_on_cpu=$(($((no_of_cpus+1))/2)) 186 numactl --all --physcpubind=$run_on_cpu support_numa pause & #just waits for sigint 187 pid=$! 188 var=`awk '{ print $2 }' /proc/$pid/stat` 189 while [ $var = '(numactl)' ]; do 190 var=`awk '{ print $2 }' /proc/$pid/stat` 191 tst_sleep 100ms 192 done 193 # Warning !! 39 represents cpu number, on which process pid is currently running and 194 # this may change if Some more fields are added in the middle, may be in future 195 running_on_cpu=$(awk '{ print $39; }' /proc/$pid/stat) 196 if [ $running_on_cpu -ne $run_on_cpu ]; then 197 tst_res TFAIL \ 198 "Process running on cpu$running_on_cpu but expected to run on cpu$run_on_cpu" 199 ROD kill -INT $pid 200 return 201 fi 202 203 ROD kill -INT $pid 204 205 tst_res TPASS "NUMA phycpubind policy" 206} 207 208# Verification of local node allocation 209test5() 210{ 211 local mem_curr 212 213 for node in $nodes_list; do 214 numactl --cpunodebind=$node --localalloc support_numa alloc_1MB & 215 pid=$! 216 217 TST_RETRY_FUNC "check_for_support_numa $pid" 0 218 219 mem_curr=$(get_mem_cur $pid $node $MB) 220 if [ $(echo "$mem_curr < $MB" |bc ) -eq 1 ]; then 221 tst_res TFAIL \ 222 "NUMA localnode memory allocated in node$node is less than expected" 223 kill -CONT $pid >/dev/null 2>&1 224 return 225 fi 226 227 kill -CONT $pid >/dev/null 2>&1 228 done 229 230 tst_res TPASS "NUMA local node allocation" 231} 232 233check_ltp_numa_test8_log() 234{ 235 grep -m1 -q '.' ltp_numa_test8.log 236} 237 238# Verification of memhog with interleave policy 239test6() 240{ 241 local mem_curr 242 # Memory will be allocated using round robin on nodes. 243 Exp_incr=$(echo "$MB / $total_nodes" |bc) 244 245 numactl --interleave=all memhog -r1000000 1MB >ltp_numa_test8.log 2>&1 & 246 pid=$! 247 248 TST_RETRY_FUNC "check_ltp_numa_test8_log" 0 249 250 for node in $nodes_list; do 251 mem_curr=$(get_mem_cur $pid $node $MB) 252 253 if [ $(echo "$mem_curr < $Exp_incr" |bc ) -eq 1 ]; then 254 tst_res TFAIL \ 255 "NUMA interleave memhog in node$node is less than expected" 256 kill -KILL $pid >/dev/null 2>&1 257 return 258 fi 259 done 260 261 kill -KILL $pid >/dev/null 2>&1 262 tst_res TPASS "NUMA MEMHOG policy" 263} 264 265# Function: hardware cheking with numa_node_size api 266# 267# Description: - Returns the size of available nodes if success. 268# 269# Input: - o/p of numactl --hardware command which is expected in the format 270# shown below 271# available: 2 nodes (0-1) 272# node 0 size: 7808 MB 273# node 0 free: 7457 MB 274# node 1 size: 5807 MB 275# node 1 free: 5731 MB 276# node distances: 277# node 0 1 278# 0: 10 20 279# 1: 20 10 280# 281test7() 282{ 283 RC=0 284 285 numactl --hardware > gavail_nodes 286 RC=$(awk '{ if ( NR == 1 ) {print $1;} }' gavail_nodes) 287 if [ $RC = "available:" ]; then 288 RC=$(awk '{ if ( NR == 1 ) {print $3;} }' gavail_nodes) 289 if [ $RC = "nodes" ]; then 290 RC=$(awk '{ if ( NR == 1 ) {print $2;} }' gavail_nodes) 291 tst_res TPASS "NUMA policy on lib NUMA_NODE_SIZE API" 292 else 293 tst_res TFAIL "Failed with NUMA policy" 294 fi 295 else 296 tst_res TFAIL "Failed with NUMA policy" 297 fi 298} 299 300# Verification of hugepage memory allocated on a node 301test8() 302{ 303 Mem_huge=0 304 Sys_node=/sys/devices/system/node 305 306 if [ ! -d "/sys/kernel/mm/hugepages/" ]; then 307 tst_res TCONF "hugepage is not supported" 308 return 309 fi 310 311 for node in $nodes_list; do 312 Ori_hpgs=$(cat ${Sys_node}/node${node}/hugepages/hugepages-${HPAGE_SIZE}kB/nr_hugepages) 313 New_hpgs=$((Ori_hpgs + 1)) 314 echo $New_hpgs >${Sys_node}/node${node}/hugepages/hugepages-${HPAGE_SIZE}kB/nr_hugepages 315 316 Chk_hpgs=$(cat ${Sys_node}/node${node}/hugepages/hugepages-${HPAGE_SIZE}kB/nr_hugepages) 317 if [ "$Chk_hpgs" -ne "$New_hpgs" ]; then 318 tst_res TCONF "hugepage is not enough to test" 319 return 320 fi 321 322 numactl --cpunodebind=$node --membind=$node support_numa alloc_1huge_page & 323 pid=$! 324 TST_RETRY_FUNC "check_for_support_numa $pid" 0 325 326 local index=$(echo "$(get_node_index $pid $node)") 327 Mem_huge=$(echo $(numastat -p $pid |awk '/^Huge/ {print $'$index'}')) 328 Mem_huge=$((${Mem_huge%.*} * 1024)) 329 330 if [ "$Mem_huge" -lt "$HPAGE_SIZE" ]; then 331 tst_res TFAIL \ 332 "NUMA memory allocated in node$node is less than expected" 333 kill -CONT $pid >/dev/null 2>&1 334 echo $Ori_hpgs >${Sys_node}/node${node}/hugepages/hugepages-${HPAGE_SIZE}kB/nr_hugepages 335 return 336 fi 337 338 kill -CONT $pid >/dev/null 2>&1 339 echo $Ori_hpgs >${Sys_node}/node${node}/hugepages/hugepages-${HPAGE_SIZE}kB/nr_hugepages 340 done 341 342 tst_res TPASS "NUMA local node hugepage memory allocated" 343} 344 345# Verification of THP memory allocated on preferred node 346test9() 347{ 348 local mem_curr 349 local cnt=1 350 351 if ! grep -q '\[always\]' /sys/kernel/mm/transparent_hugepage/enabled; then 352 tst_res TCONF "THP is not supported/enabled" 353 return 354 fi 355 356 for node in $nodes_list; do 357 if [ $cnt -eq $total_nodes ]; then #wrap up for last node 358 Preferred_node=$(echo $nodes_list | cut -d ' ' -f 1) 359 else 360 # always next node is preferred node 361 Preferred_node=$(echo $nodes_list | cut -d ' ' -f $((cnt+1))) 362 fi 363 364 numactl --cpunodebind=$node --preferred=$Preferred_node support_numa alloc_2HPSZ_THP & 365 pid=$! 366 367 TST_RETRY_FUNC "check_for_support_numa $pid" 0 368 369 mem_curr=$(get_mem_cur $pid $Preferred_node 1024) 370 if [ $(echo "$mem_curr < $HPAGE_SIZE * 2" |bc ) -eq 1 ]; then 371 tst_res TFAIL \ 372 "NUMA memory allocated in node$Preferred_node is less than expected" 373 kill -CONT $pid >/dev/null 2>&1 374 return 375 fi 376 377 cnt=$((cnt+1)) 378 kill -CONT $pid >/dev/null 2>&1 379 done 380 381 tst_res TPASS "NUMA preferred node policy verified with THP enabled" 382} 383 384. tst_test.sh 385tst_run 386