Can you please specify how you set the power?
Sure, the power limit can be changed using nvidia-smi
:
sudo nvidia-smi -pl 260
or calling the nvmlDeviceSetPowerManagementLimit()
form the nvml library:
* For all discrete products with dedicated fans.
*
* The fan speed is expressed as a percentage of the product's maximum noise tolerance fan speed.
* This value may exceed 100% in certain cases.
*
* @param device The identifier of the target device
* @param fan The index of the target fan, zero indexed.
* @param speed Reference in which to return the fan speed percentage
*
* @return
* - \ref NVML_SUCCESS if \a speed has been set
* - \ref NVML_ERROR_UNINITIALIZED if the library has not been successfully initialized
* - \ref NVML_ERROR_INVALID_ARGUMENT if \a device is invalid, \a fan is not an acceptable index, or \a speed is NULL
* - \ref NVML_ERROR_NOT_SUPPORTED if the device does not have a fan or is newer than Maxwell
* - \ref NVML_ERROR_GPU_IS_LOST if the target GPU has fallen off the bus or is otherwise inaccessible
* - \ref NVML_ERROR_UNKNOWN on any unexpected error
*/
nvmlReturn_t DECLDIR nvmlDeviceGetFanSpeed_v2(nvmlDevice_t device, unsigned int fan, unsigned int * speed);
/**
* Retrieves the intended target speed of the device's specified fan.
Which specific files do you write to? Ideally a strace of a successful operation could be attached.
execve("/usr/bin/nvidia-smi", ["nvidia-smi", "-pl", "261"], 0x7ffd8e3ce9f0 /* 22 vars */) = 0
brk(NULL) = 0xe0a000
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=179570, ...}) = 0
mmap(NULL, 179570, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fa263044000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0`l\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=149696, ...}) = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fa263042000
mmap(NULL, 132288, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fa263021000
mmap(0x7fa263027000, 61440, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x7fa263027000
mmap(0x7fa263036000, 24576, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x15000) = 0x7fa263036000
mmap(0x7fa26303c000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1a000) = 0x7fa26303c000
mmap(0x7fa26303e000, 13504, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fa26303e000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0000\21\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=18656, ...}) = 0
mmap(NULL, 20752, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fa26301b000
mmap(0x7fa26301c000, 8192, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1000) = 0x7fa26301c000
mmap(0x7fa26301e000, 4096, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7fa26301e000
mmap(0x7fa26301f000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7fa26301f000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260A\2\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1996592, ...}) = 0
mmap(NULL, 2004992, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fa262e31000
mprotect(0x7fa262e53000, 1826816, PROT_NONE) = 0
mmap(0x7fa262e53000, 1511424, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x22000) = 0x7fa262e53000
mmap(0x7fa262fc4000, 311296, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x193000) = 0x7fa262fc4000
mmap(0x7fa263011000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1df000) = 0x7fa263011000
mmap(0x7fa263017000, 14336, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fa263017000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/librt.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260#\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=35776, ...}) = 0
mmap(NULL, 39904, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fa262e27000
mmap(0x7fa262e29000, 16384, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7fa262e29000
mmap(0x7fa262e2d000, 8192, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x7fa262e2d000
mmap(0x7fa262e2f000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x7000) = 0x7fa262e2f000
close(3) = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fa262e25000
arch_prctl(ARCH_SET_FS, 0x7fa262e25b80) = 0
mprotect(0x7fa263011000, 16384, PROT_READ) = 0
mprotect(0x7fa26303c000, 4096, PROT_READ) = 0
mprotect(0x7fa262e2f000, 4096, PROT_READ) = 0
mprotect(0x7fa26301f000, 4096, PROT_READ) = 0
mprotect(0x7fa263099000, 4096, PROT_READ) = 0
munmap(0x7fa263044000, 179570) = 0
set_tid_address(0x7fa262e25e50) = 25259
set_robust_list(0x7fa262e25e60, 24) = 0
rt_sigaction(SIGRTMIN, {sa_handler=0x7fa2630276c0, sa_mask=[], sa_flags=SA_RESTORER|SA_SIGINFO, sa_restorer=0x7fa263033dd0}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {sa_handler=0x7fa263027750, sa_mask=[], sa_flags=SA_RESTORER|SA_RESTART|SA_SIGINFO, sa_restorer=0x7fa263033dd0}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
prlimit64(0, RLIMIT_STACK, NULL, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0
futex(0x7fa2630200c8, FUTEX_WAKE_PRIVATE, 2147483647) = 0
brk(NULL) = 0xe0a000
brk(0xe2b000) = 0xe2b000
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=179570, ...}) = 0
mmap(NULL, 179570, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fa263044000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib/x86_64-linux-gnu/libnvidia-ml.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\r\1\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=1438616, ...}) = 0
mmap(NULL, 6459240, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fa2627fc000
mprotect(0x7fa262944000, 2093056, PROT_NONE) = 0
mmap(0x7fa262b43000, 98304, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x147000) = 0x7fa262b43000
mmap(0x7fa262b5b000, 2924392, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fa262b5b000
close(3) = 0
munmap(0x7fa263044000, 179570) = 0
getpid() = 25259
openat(AT_FDCWD, "/proc/modules", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
read(3, "ip6t_MASQUERADE 16384 1 - Live 0"..., 1024) = 1024
read(3, "84 6 - Live 0xffffffffc0d11000\nn"..., 1024) = 1024
read(3, "e 0xffffffffc0a99000 (OE)\nvboxdr"..., 1024) = 1024
read(3, "000\nnvidia_drm 40960 8 - Live 0x"..., 1024) = 1024
close(3) = 0
openat(AT_FDCWD, "/proc/driver/nvidia/params", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
read(3, "Mobile: 4294967295\nResmanDebugLe"..., 1024) = 649
close(3) = 0
stat("/dev/nvidiactl", {st_mode=S_IFCHR|0666, st_rdev=makedev(195, 255), ...}) = 0
openat(AT_FDCWD, "/dev/nvidiactl", O_RDWR) = 3
fcntl(3, F_SETFD, FD_CLOEXEC) = 0
ioctl(3, _IOC(_IOC_READ|_IOC_WRITE, 0x46, 0xd2, 0x48), 0x7ffd19b72830) = 0
openat(AT_FDCWD, "/sys/devices/system/memory/block_size_bytes", O_RDONLY) = 4
read(4, "8000000\n", 99) = 8
close(4) = 0
ioctl(3, _IOC(_IOC_READ|_IOC_WRITE, 0x46, 0xd6, 0x8), 0x7ffd19b72830) = 0
ioctl(3, _IOC(_IOC_READ|_IOC_WRITE, 0x46, 0xca, 0x4), 0x7fa262e23d00) = 0
ioctl(3, _IOC(_IOC_READ|_IOC_WRITE, 0x46, 0xc8, 0xa00), 0x7fa262e23d60) = 0
ioctl(3, _IOC(_IOC_READ|_IOC_WRITE, 0x46, 0x2b, 0x20), 0x7ffd19b72880) = 0
ioctl(3, _IOC(_IOC_READ|_IOC_WRITE, 0x46, 0x2a, 0x20), 0x7ffd19b72870) = 0
ioctl(3, _IOC(_IOC_READ|_IOC_WRITE, 0x46, 0x2a, 0x20), 0x7ffd19b72870) = 0
ioctl(3, _IOC(_IOC_READ|_IOC_WRITE, 0x46, 0x2a, 0x20), 0x7ffd19b72870) = 0
ioctl(3, _IOC(_IOC_READ|_IOC_WRITE, 0x46, 0x2a, 0x20), 0x7ffd19b72870) = 0
openat(AT_FDCWD, "/proc/driver/nvidia/params", O_RDONLY) = 4
fstat(4, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
read(4, "Mobile: 4294967295\nResmanDebugLe"..., 1024) = 649
close(4) = 0
stat("/dev/nvidia0", {st_mode=S_IFCHR|0666, st_rdev=makedev(195, 0), ...}) = 0
openat(AT_FDCWD, "/dev/nvidia0", O_RDWR) = 4
fcntl(4, F_SETFD, FD_CLOEXEC) = 0
ioctl(3, _IOC(_IOC_READ|_IOC_WRITE, 0x46, 0x2a, 0x20), 0x7ffd19b72870) = 0
getpid() = 25259
getpid() = 25259
getpid() = 25259
ioctl(3, _IOC(_IOC_READ|_IOC_WRITE, 0x46, 0x2a, 0x20), 0x7ffd19b74de0) = 0
ioctl(3, _IOC(_IOC_READ|_IOC_WRITE, 0x46, 0x2a, 0x20), 0x7ffd19b74c00) = 0
ioctl(3, _IOC(_IOC_READ|_IOC_WRITE, 0x46, 0x2a, 0x20), 0x7ffd19b74af0) = 0
ioctl(3, _IOC(_IOC_READ|_IOC_WRITE, 0x46, 0x2a, 0x20), 0x7ffd19b74af0) = 0
openat(AT_FDCWD, "/proc/driver/nvidia/params", O_RDONLY) = 5
fstat(5, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
read(5, "Mobile: 4294967295\nResmanDebugLe"..., 1024) = 649
close(5) = 0
stat("/dev/nvidia0", {st_mode=S_IFCHR|0666, st_rdev=makedev(195, 0), ...}) = 0
openat(AT_FDCWD, "/dev/nvidia0", O_RDWR) = 5
fcntl(5, F_SETFD, FD_CLOEXEC) = 0
ioctl(5, _IOC(_IOC_READ|_IOC_WRITE, 0x46, 0xd7, 0x228), 0x7ffd19b74a20) = 0
ioctl(3, _IOC(_IOC_READ|_IOC_WRITE, 0x46, 0x2b, 0x20), 0x7ffd19b74cd0) = 0
ioctl(3, _IOC(_IOC_READ|_IOC_WRITE, 0x46, 0x2a, 0x20), 0x7ffd19b74d60) = 0
ioctl(3, _IOC(_IOC_READ|_IOC_WRITE, 0x46, 0x2a, 0x20), 0x7ffd19b74ac0) = 0
ioctl(3, _IOC(_IOC_READ|_IOC_WRITE, 0x46, 0x2a, 0x20), 0x7ffd19b749c0) = 0
ioctl(3, _IOC(_IOC_READ|_IOC_WRITE, 0x46, 0x2a, 0x20), 0x7ffd19b749c0) = 0
openat(AT_FDCWD, "/proc/driver/nvidia/params", O_RDONLY) = 6
fstat(6, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
read(6, "Mobile: 4294967295\nResmanDebugLe"..., 1024) = 649
close(6) = 0
stat("/dev/nvidia0", {st_mode=S_IFCHR|0666, st_rdev=makedev(195, 0), ...}) = 0
openat(AT_FDCWD, "/dev/nvidia0", O_RDWR) = 6
fcntl(6, F_SETFD, FD_CLOEXEC) = 0
ioctl(3, _IOC(_IOC_READ|_IOC_WRITE, 0x46, 0x2b, 0x20), 0x7ffd19b74b80) = 0
ioctl(3, _IOC(_IOC_READ|_IOC_WRITE, 0x46, 0x2a, 0x20), 0x7ffd19b74bf0) = 0
nanosleep({tv_sec=0, tv_nsec=5000000}, NULL) = 0
ioctl(3, _IOC(_IOC_READ|_IOC_WRITE, 0x46, 0x2a, 0x20), 0x7ffd19b74bf0) = 0
getpid() = 25259
ioctl(3, _IOC(_IOC_READ|_IOC_WRITE, 0x46, 0x2a, 0x20), 0x7ffd19b74f20) = 0
getpid() = 25259
ioctl(3, _IOC(_IOC_READ|_IOC_WRITE, 0x46, 0x2a, 0x20), 0x7ffd19b74da0) = 0
ioctl(3, _IOC(_IOC_READ|_IOC_WRITE, 0x46, 0x2a, 0x20), 0x7ffd19b74da0) = 0
ioctl(3, _IOC(_IOC_READ|_IOC_WRITE, 0x46, 0x2a, 0x20), 0x7ffd19b74ce0) = 0
ioctl(3, _IOC(_IOC_READ|_IOC_WRITE, 0x46, 0x2a, 0x20), 0x7ffd19b74e20) = 0
ioctl(3, _IOC(_IOC_READ|_IOC_WRITE, 0x46, 0x2a, 0x20), 0x7ffd19b74e80) = 0
getpid() = 25259
ioctl(3, _IOC(_IOC_READ|_IOC_WRITE, 0x46, 0x2a, 0x20), 0x7ffd19b731b0) = 0
ioctl(3, _IOC(_IOC_READ|_IOC_WRITE, 0x46, 0x2a, 0x20), 0x7ffd19b73220) = 0
getpid() = 25259
geteuid() = 0
ioctl(3, _IOC(_IOC_READ|_IOC_WRITE, 0x46, 0x2a, 0x20), 0x7ffd19b74e80) = 0
fstat(1, {st_mode=S_IFREG|0644, st_size=11022, ...}) = 0
getpid() = 25259
stat("/var/run/nvidia-persistenced/socket", {st_mode=S_IFSOCK|0777, st_size=0, ...}) = 0
socket(AF_UNIX, SOCK_STREAM, 0) = 7
connect(7, {sa_family=AF_UNIX, sun_path="/var/run/nvidia-persistenced/socket"}, 38) = 0
getpid() = 25259
futex(0x7fa263019c00, FUTEX_WAKE_PRIVATE, 2147483647) = 0
getpid() = 25259
geteuid() = 0
getegid() = 0
sendmsg(7, {msg_name=NULL, msg_namelen=0, msg_iov=[{iov_base="\200\0\0008)\366\235\331\0\0\0\0\0\0\0\2\0\0\210\276\0\0\0\1\0\0\0\2\0\0\0\0"..., iov_len=60}], msg_iovlen=1, msg_control=[{cmsg_len=28, cmsg_level=SOL_SOCKET, cmsg_type=SCM_CREDENTIALS, cmsg_data={pid=25259, uid=0, gid=0}}], msg_controllen=32, msg_flags=0}, 0) = 60
poll([{fd=7, events=POLLIN}], 1, 25000) = 1 ([{fd=7, revents=POLLIN}])
setsockopt(7, SOL_SOCKET, SO_PASSCRED, [1], 4) = 0
recvmsg(7, {msg_name=NULL, msg_namelen=0, msg_iov=[{iov_base="\200\0\0 )\366\235\331\0\0\0\1\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., iov_len=4000}], msg_iovlen=1, msg_control=[{cmsg_len=28, cmsg_level=SOL_SOCKET, cmsg_type=SCM_CREDENTIALS, cmsg_data={pid=4607, uid=126, gid=132}}], msg_controllen=32, msg_flags=0}, 0) = 36
close(7) = 0
ioctl(3, _IOC(_IOC_READ|_IOC_WRITE, 0x46, 0x2a, 0x20), 0x7ffd19b74e70) = 0
ioctl(3, _IOC(_IOC_READ|_IOC_WRITE, 0x46, 0x29, 0x10), 0x7ffd19b74de0) = 0
close(6) = 0
ioctl(3, _IOC(_IOC_READ|_IOC_WRITE, 0x46, 0x29, 0x10), 0x7ffd19b74de0) = 0
close(5) = 0
ioctl(3, _IOC(_IOC_READ|_IOC_WRITE, 0x46, 0x2a, 0x20), 0x7ffd19b74e20) = 0
close(4) = 0
ioctl(3, _IOC(_IOC_READ|_IOC_WRITE, 0x46, 0x29, 0x10), 0x7ffd19b74e50) = 0
close(3) = 0
write(1, "Power limit for GPU 00000000:0B:"..., 363Power limit for GPU 00000000:0B:00.0 was set to 261.00 W from 260.00 W.
Warning: persistence mode is disabled on this device. This settings will go back to default as soon as driver unloads (e.g. last application like nvidia-smi or cuda application terminates). Run with [--help | -h] switch to get more information on how to enable persistence mode.
All done.
) = 363
exit_group(0) = ?
+++ exited with 0 +++
Let me know if I can provide any more info