1#!/bin/sh 2############################################################################## 3# # 4# Copyright (c) International Business Machines Corp., 2007 # 5# Sivakumar Chinnaiah, Sivakumar.C@in.ibm.com # 6# Copyright (c) Linux Test Project, 2016 # 7# # 8# This program is free software: you can redistribute it and/or modify # 9# it under the terms of the GNU General Public License as published by # 10# the Free Software Foundation, either version 3 of the License, or # 11# (at your option) any later version. # 12# # 13# This program is distributed in the hope that it will be useful, # 14# but WITHOUT ANY WARRANTY; without even the implied warranty of # 15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # 16# GNU General Public License for more details. # 17# # 18# You should have received a copy of the GNU General Public License # 19# along with this program. If not, see <http://www.gnu.org/licenses/>. # 20# # 21############################################################################## 22# # 23# Description: Test Basic functionality of numactl command. # 24# Test #1: Verifies cpunodebind and membind # 25# Test #2: Verifies preferred node bind for memory allocation # 26# Test #3: Verifies share memory allocation on preferred node # 27# Test #4: Verifies memory interleave on all nodes # 28# Test #5: Verifies share memory interleave on all nodes # 29# Test #6: Verifies physcpubind # 30# Test #7: Verifies localalloc # 31# Test #8: Verifies memhog # 32# Test #9: Verifies numa_node_size api # 33# Test #10:Verifies Migratepages # 34# Test #11:Verifies hugepage alloacted on specified node # 35# Test #12:Verifies THP memory allocated on preferred node # 36# # 37############################################################################## 38 39TST_CNT=12 40TST_SETUP=setup 41TST_TESTFUNC=test 42TST_NEEDS_TMPDIR=1 43TST_NEEDS_ROOT=1 44TST_NEEDS_CMDS="numactl numastat awk" 45 46. tst_test.sh 47 48# 49# Extracts the value of given numa node from the `numastat -p` output. 50# 51# $1 - Pid number. 52# $2 - Node number. 53# 54extract_numastat_p() 55{ 56 local pid=$1 57 local node=$(($2 + 2)) 58 59 echo $(numastat -p $pid |awk '/^Total/ {print $'$node'}') 60} 61 62check_for_support_numa() 63{ 64 local pid=$1 65 66 local state=$(awk '{print $3}' /proc/$pid/stat) 67 68 if [ $state = 'T' ]; then 69 return 0 70 fi 71 72 return 1 73} 74 75setup() 76{ 77 export MB=$((1024*1024)) 78 export PAGE_SIZE=$(tst_getconf PAGESIZE) 79 export HPAGE_SIZE=$(awk '/Hugepagesize:/ {print $2}' /proc/meminfo) 80 81 total_nodes=0 82 83 nodes_list=$(numactl --show | grep nodebind | cut -d ':' -f 2) 84 for node in $nodes_list; do 85 total_nodes=$((total_nodes+1)) 86 done 87 88 tst_res TINFO "The system contains $total_nodes nodes: $nodes_list" 89 if [ $total_nodes -le 1 ]; then 90 tst_brk TCONF "your machine does not support numa policy 91 or your machine is not a NUMA machine" 92 fi 93} 94 95# Verification of memory allocated on a node 96test1() 97{ 98 Mem_curr=0 99 100 for node in $nodes_list; do 101 numactl --cpunodebind=$node --membind=$node support_numa alloc_1MB & 102 pid=$! 103 104 TST_RETRY_FUNC "check_for_support_numa $pid" 0 105 106 Mem_curr=$(echo "$(extract_numastat_p $pid $node) * $MB" |bc) 107 if [ $(echo "$Mem_curr < $MB" | bc) -eq 1 ]; then 108 tst_res TFAIL \ 109 "NUMA memory allocated in node$node is less than expected" 110 kill -CONT $pid >/dev/null 2>&1 111 return 112 fi 113 114 kill -CONT $pid >/dev/null 2>&1 115 done 116 117 tst_res TPASS "NUMA local node and memory affinity" 118} 119 120# Verification of memory allocated on preferred node 121test2() 122{ 123 Mem_curr=0 124 125 COUNTER=1 126 for node in $nodes_list; do 127 128 if [ $COUNTER -eq $total_nodes ]; then #wrap up for last node 129 Preferred_node=$(echo $nodes_list | cut -d ' ' -f 1) 130 else 131 # always next node is preferred node 132 Preferred_node=$(echo $nodes_list | cut -d ' ' -f $((COUNTER+1))) 133 fi 134 135 numactl --cpunodebind=$node --preferred=$Preferred_node support_numa alloc_1MB & 136 pid=$! 137 138 TST_RETRY_FUNC "check_for_support_numa $pid" 0 139 140 Mem_curr=$(echo "$(extract_numastat_p $pid $Preferred_node) * $MB" |bc) 141 if [ $(echo "$Mem_curr < $MB" |bc ) -eq 1 ]; then 142 tst_res TFAIL \ 143 "NUMA memory allocated in node$Preferred_node is less than expected" 144 kill -CONT $pid >/dev/null 2>&1 145 return 146 fi 147 148 COUNTER=$((COUNTER+1)) 149 kill -CONT $pid >/dev/null 2>&1 150 done 151 152 tst_res TPASS "NUMA preferred node policy" 153} 154 155# Verification of share memory allocated on preferred node 156test3() 157{ 158 Mem_curr=0 159 COUNTER=1 160 161 for node in $nodes_list; do 162 163 if [ $COUNTER -eq $total_nodes ] #wrap up for last node 164 then 165 Preferred_node=$(echo $nodes_list | cut -d ' ' -f 1) 166 else 167 # always next node is preferred node 168 Preferred_node=$(echo $nodes_list | cut -d ' ' -f $((COUNTER+1))) 169 fi 170 171 numactl --cpunodebind=$node --preferred=$Preferred_node support_numa alloc_1MB_shared & 172 pid=$! 173 174 TST_RETRY_FUNC "check_for_support_numa $pid" 0 175 176 Mem_curr=$(echo "$(extract_numastat_p $pid $Preferred_node) * $MB" |bc) 177 if [ $(echo "$Mem_curr < $MB" |bc ) -eq 1 ]; then 178 tst_res TFAIL \ 179 "NUMA share memory allocated in node$Preferred_node is less than expected" 180 kill -CONT $pid >/dev/null 2>&1 181 return 182 fi 183 184 COUNTER=$((COUNTER+1)) 185 kill -CONT $pid >/dev/null 2>&1 186 done 187 188 tst_res TPASS "NUMA share memory allocated in preferred node" 189} 190 191# Verification of memory interleaved on all nodes 192test4() 193{ 194 Mem_curr=0 195 # Memory will be allocated using round robin on nodes. 196 Exp_incr=$(echo "$MB / $total_nodes" |bc) 197 198 numactl --interleave=all support_numa alloc_1MB & 199 pid=$! 200 201 TST_RETRY_FUNC "check_for_support_numa $pid" 0 202 203 for node in $nodes_list; do 204 Mem_curr=$(echo "$(extract_numastat_p $pid $node) * $MB" |bc) 205 206 if [ $(echo "$Mem_curr < $Exp_incr" |bc ) -eq 1 ]; then 207 tst_res TFAIL \ 208 "NUMA interleave memory allocated in node$node is less than expected" 209 kill -CONT $pid >/dev/null 2>&1 210 return 211 fi 212 done 213 214 kill -CONT $pid >/dev/null 2>&1 215 tst_res TPASS "NUMA interleave policy" 216} 217 218# Verification of shared memory interleaved on all nodes 219test5() 220{ 221 Mem_curr=0 222 # Memory will be allocated using round robin on nodes. 223 Exp_incr=$(echo "$MB / $total_nodes" |bc) 224 225 numactl --interleave=all support_numa alloc_1MB_shared & 226 pid=$! 227 228 TST_RETRY_FUNC "check_for_support_numa $pid" 0 229 230 for node in $nodes_list; do 231 Mem_curr=$(echo "$(extract_numastat_p $pid $node) * $MB" |bc) 232 233 if [ $(echo "$Mem_curr < $Exp_incr" |bc ) -eq 1 ]; then 234 tst_res TFAIL \ 235 "NUMA interleave share memory allocated in node$node is less than expected" 236 kill -CONT $pid >/dev/null 2>&1 237 return 238 fi 239 done 240 241 kill -CONT $pid >/dev/null 2>&1 242 243 tst_res TPASS "NUMA interleave policy on shared memory" 244} 245 246# Verification of physical cpu bind 247test6() 248{ 249 no_of_cpus=0 #no. of cpu's exist 250 run_on_cpu=0 251 running_on_cpu=0 252 253 no_of_cpus=$(tst_ncpus) 254 # not sure whether cpu's can't be in odd number 255 run_on_cpu=$(($((no_of_cpus+1))/2)) 256 numactl --physcpubind=$run_on_cpu support_numa pause & #just waits for sigint 257 pid=$! 258 var=`awk '{ print $2 }' /proc/$pid/stat` 259 while [ $var = '(numactl)' ]; do 260 var=`awk '{ print $2 }' /proc/$pid/stat` 261 tst_sleep 100ms 262 done 263 # Warning !! 39 represents cpu number, on which process pid is currently running and 264 # this may change if Some more fields are added in the middle, may be in future 265 running_on_cpu=$(awk '{ print $39; }' /proc/$pid/stat) 266 if [ $running_on_cpu -ne $run_on_cpu ]; then 267 tst_res TFAIL \ 268 "Process running on cpu$running_on_cpu but expected to run on cpu$run_on_cpu" 269 ROD kill -INT $pid 270 return 271 fi 272 273 ROD kill -INT $pid 274 275 tst_res TPASS "NUMA phycpubind policy" 276} 277 278# Verification of local node allocation 279test7() 280{ 281 Mem_curr=0 282 283 for node in $nodes_list; do 284 numactl --cpunodebind=$node --localalloc support_numa alloc_1MB & 285 pid=$! 286 287 TST_RETRY_FUNC "check_for_support_numa $pid" 0 288 289 Mem_curr=$(echo "$(extract_numastat_p $pid $node) * $MB" |bc) 290 if [ $(echo "$Mem_curr < $MB" |bc ) -eq 1 ]; then 291 tst_res TFAIL \ 292 "NUMA localnode memory allocated in node$node is less than expected" 293 kill -CONT $pid >/dev/null 2>&1 294 return 295 fi 296 297 kill -CONT $pid >/dev/null 2>&1 298 done 299 300 tst_res TPASS "NUMA local node allocation" 301} 302 303check_ltp_numa_test8_log() 304{ 305 grep -m1 -q '.' ltp_numa_test8.log 306} 307 308# Verification of memhog with interleave policy 309test8() 310{ 311 Mem_curr=0 312 # Memory will be allocated using round robin on nodes. 313 Exp_incr=$(echo "$MB / $total_nodes" |bc) 314 315 numactl --interleave=all memhog -r1000000 1MB >ltp_numa_test8.log 2>&1 & 316 pid=$! 317 318 TST_RETRY_FUNC "check_ltp_numa_test8_log" 0 319 320 for node in $nodes_list; do 321 Mem_curr=$(echo "$(extract_numastat_p $pid $node) * $MB" |bc) 322 323 if [ $(echo "$Mem_curr < $Exp_incr" |bc ) -eq 1 ]; then 324 tst_res TFAIL \ 325 "NUMA interleave memhog in node$node is less than expected" 326 kill -KILL $pid >/dev/null 2>&1 327 return 328 fi 329 done 330 331 kill -KILL $pid >/dev/null 2>&1 332 tst_res TPASS "NUMA MEMHOG policy" 333} 334 335# Function: hardware cheking with numa_node_size api 336# 337# Description: - Returns the size of available nodes if success. 338# 339# Input: - o/p of numactl --hardware command which is expected in the format 340# shown below 341# available: 2 nodes (0-1) 342# node 0 size: 7808 MB 343# node 0 free: 7457 MB 344# node 1 size: 5807 MB 345# node 1 free: 5731 MB 346# node distances: 347# node 0 1 348# 0: 10 20 349# 1: 20 10 350# 351test9() 352{ 353 RC=0 354 355 numactl --hardware > gavail_nodes 356 RC=$(awk '{ if ( NR == 1 ) {print $1;} }' gavail_nodes) 357 if [ $RC = "available:" ]; then 358 RC=$(awk '{ if ( NR == 1 ) {print $3;} }' gavail_nodes) 359 if [ $RC = "nodes" ]; then 360 RC=$(awk '{ if ( NR == 1 ) {print $2;} }' gavail_nodes) 361 tst_res TPASS "NUMA policy on lib NUMA_NODE_SIZE API" 362 else 363 tst_res TFAIL "Failed with numa policy" 364 fi 365 else 366 tst_res TFAIL "Failed with numa policy" 367 fi 368} 369 370# Verification of migratepages 371test10() 372{ 373 Mem_curr=0 374 COUNTER=1 375 376 for node in $nodes_list; do 377 378 if [ $COUNTER -eq $total_nodes ]; then 379 Preferred_node=$(echo $nodes_list | cut -d ' ' -f 1) 380 else 381 Preferred_node=$(echo $nodes_list | cut -d ' ' -f $((COUNTER+1))) 382 fi 383 384 numactl --preferred=$node support_numa alloc_1MB & 385 pid=$! 386 387 TST_RETRY_FUNC "check_for_support_numa $pid" 0 388 389 migratepages $pid $node $Preferred_node 390 391 Mem_curr=$(echo "$(extract_numastat_p $pid $Preferred_node) * $MB" |bc) 392 if [ $(echo "$Mem_curr < $MB" |bc ) -eq 1 ]; then 393 tst_res TFAIL \ 394 "NUMA migratepages is not working fine" 395 kill -CONT $pid >/dev/null 2>&1 396 return 397 fi 398 399 COUNTER=$((COUNTER+1)) 400 kill -CONT $pid >/dev/null 2>&1 401 done 402 403 tst_res TPASS "NUMA MIGRATEPAGES policy" 404} 405 406# Verification of hugepage memory allocated on a node 407test11() 408{ 409 Mem_huge=0 410 Sys_node=/sys/devices/system/node 411 412 if [ ! -d "/sys/kernel/mm/hugepages/" ]; then 413 tst_res TCONF "hugepage is not supported" 414 return 415 fi 416 417 for node in $nodes_list; do 418 Ori_hpgs=$(cat ${Sys_node}/node${node}/hugepages/hugepages-${HPAGE_SIZE}kB/nr_hugepages) 419 New_hpgs=$((Ori_hpgs + 1)) 420 echo $New_hpgs >${Sys_node}/node${node}/hugepages/hugepages-${HPAGE_SIZE}kB/nr_hugepages 421 422 Chk_hpgs=$(cat ${Sys_node}/node${node}/hugepages/hugepages-${HPAGE_SIZE}kB/nr_hugepages) 423 if [ "$Chk_hpgs" -ne "$New_hpgs" ]; then 424 tst_res TCONF "hugepage is not enough to test" 425 return 426 fi 427 428 numactl --cpunodebind=$node --membind=$node support_numa alloc_1huge_page & 429 pid=$! 430 TST_RETRY_FUNC "check_for_support_numa $pid" 0 431 432 Mem_huge=$(echo $(numastat -p $pid |awk '/^Huge/ {print $'$((node+2))'}')) 433 Mem_huge=$((${Mem_huge%.*} * 1024)) 434 435 if [ "$Mem_huge" -lt "$HPAGE_SIZE" ]; then 436 tst_res TFAIL \ 437 "NUMA memory allocated in node$node is less than expected" 438 kill -CONT $pid >/dev/null 2>&1 439 echo $Ori_hpgs >${Sys_node}/node${node}/hugepages/hugepages-${HPAGE_SIZE}kB/nr_hugepages 440 return 441 fi 442 443 kill -CONT $pid >/dev/null 2>&1 444 echo $Ori_hpgs >${Sys_node}/node${node}/hugepages/hugepages-${HPAGE_SIZE}kB/nr_hugepages 445 done 446 447 tst_res TPASS "NUMA local node hugepage memory allocated" 448} 449 450# Verification of THP memory allocated on preferred node 451test12() 452{ 453 Mem_curr=0 454 455 if ! grep -q '\[always\]' /sys/kernel/mm/transparent_hugepage/enabled; then 456 tst_res TCONF "THP is not supported/enabled" 457 return 458 fi 459 460 COUNTER=1 461 for node in $nodes_list; do 462 463 if [ $COUNTER -eq $total_nodes ]; then #wrap up for last node 464 Preferred_node=$(echo $nodes_list | cut -d ' ' -f 1) 465 else 466 # always next node is preferred node 467 Preferred_node=$(echo $nodes_list | cut -d ' ' -f $((COUNTER+1))) 468 fi 469 470 numactl --cpunodebind=$node --preferred=$Preferred_node support_numa alloc_2HPSZ_THP & 471 pid=$! 472 473 TST_RETRY_FUNC "check_for_support_numa $pid" 0 474 475 Mem_curr=$(echo "$(extract_numastat_p $pid $Preferred_node) * 1024" |bc) 476 if [ $(echo "$Mem_curr < $HPAGE_SIZE * 2" |bc ) -eq 1 ]; then 477 tst_res TFAIL \ 478 "NUMA memory allocated in node$Preferred_node is less than expected" 479 kill -CONT $pid >/dev/null 2>&1 480 return 481 fi 482 483 COUNTER=$((COUNTER+1)) 484 kill -CONT $pid >/dev/null 2>&1 485 done 486 487 tst_res TPASS "NUMA preferred node policy verified with THP enabled" 488} 489 490tst_run 491