Cannot locate base snap core18: Permission denied

quater · November 5, 2020, 12:22am

Commands for launching container and accessing it as root.

$ export LXD_CONTAINER_NAME="container1"

$ lxc launch ubuntu:20.10 --profile default $LXD_CONTAINER_NAME                         
Creating container1
Starting container1

$ lxc exec $LXD_CONTAINER_NAME -- /bin/bash   

root@container1:~# ls -lah /snap/
total 7.0K
drwxr-xr-x  6 root root   7 Nov  3 11:33 .
drwxr-xr-x 18 root root  24 Nov  3 11:37 ..
-r--r--r--  1 root root 548 Nov  3 11:33 README
drwxr-xr-x  2 root root  10 Nov  3 11:33 bin
drwxr-xr-x  3 root root   4 Nov  3 11:33 core18
drwxr-xr-x  3 root root   4 Nov  3 11:33 lxd
drwxr-xr-x  3 root root   4 Nov  3 11:33 snapd

root@container1:~# ls -lah /snap/core18/
total 2.5K
drwxr-xr-x  3 root root 4 Nov  3 11:33 .
drwxr-xr-x  6 root root 7 Nov  3 11:33 ..
drwxr-xr-x 24 root root 0 Sep 29 10:45 1932
lrwxrwxrwx  1 root root 4 Nov  3 11:33 current -> 1932

root@container1:~# ls -lah /snap/core18/1932/
total 0
lrwxrwxrwx  1 root root 15 Sep 29 10:44 .disk -> /writable/.disk
drwxr-xr-x  2 root root  0 Sep 29 10:45 bin
drwxr-xr-x  6 root root  0 Sep 29 10:45 boot
drwxr-xr-x  2 root root  0 Sep 29 10:45 dev
drwxr-xr-x 42 root root  0 Sep 29 10:45 etc
drwxr-xr-x  2 root root  0 Apr 24  2018 home
drwxr-xr-x 14 root root  0 Sep 29 10:45 lib
drwxr-xr-x  2 root root  0 Sep 29 10:45 lib64
drwxr-xr-x  2 root root  0 Aug  6 22:33 media
drwxr-xr-x  2 root root  0 Sep 29 10:45 meta
drwxr-xr-x  2 root root  0 Aug  6 22:33 mnt
drwxr-xr-x  2 root root  0 Aug  6 22:33 opt
drwxr-xr-x  2 root root  0 Apr 24  2018 proc
drwx------  2 root root  0 Sep 29 10:45 root
drwxr-xr-x  2 root root  0 Sep 29 10:44 run
drwxr-xr-x  2 root root  0 Sep 29 10:45 sbin
drwxr-xr-x  2 root root  0 Sep 29 10:45 snap
drwxr-xr-x  2 root root  0 Aug  6 22:33 srv
-rw-r--r--  1 root root 53 Sep 29 10:45 stdout
drwxr-xr-x  2 root root  0 Apr 24  2018 sys
drwxrwxrwt  2 root root  0 Sep 29 10:45 tmp
drwxr-xr-x 11 root root  0 Sep 29 10:45 usr
drwxr-xr-x 12 root root  0 Sep 29 10:45 var
drwxr-xr-x  2 root root  0 Sep 29 10:44 writable

root@container1:~# ps fauxww
USER         PID %CPU %MEM    VSZ   RSS TTY      STAT START   TIME COMMAND
root         702  0.0  0.0   8952  2844 pts/0    Ss   00:14   0:00 /bin/bash
root         709  0.0  0.0  11476  2200 pts/0    R+   00:14   0:00  \_ ps fauxww
root           1  0.0  0.0 170676  8228 ?        Ss   00:07   0:00 /sbin/init
root          62  0.0  0.0  36240  9528 ?        Ss   00:07   0:00 /lib/systemd/systemd-journald
root          94  0.0  0.0  20452  3364 ?        Ss   00:07   0:00 /lib/systemd/systemd-udevd
root         113  0.0  0.0   3692  1236 ?        Ss   00:07   0:00 snapfuse /var/lib/snapd/snaps/lxd_18137.snap /snap/lxd/18137 -o ro,nodev,suid
root         114  0.0  0.0   3624  1188 ?        Ss   00:07   0:00 snapfuse /var/lib/snapd/snaps/snapd_9721.snap /snap/snapd/9721 -o ro,nodev,suid
root         115  0.0  0.0   3732  1412 ?        Ss   00:07   0:00 snapfuse /var/lib/snapd/snaps/core18_1932.snap /snap/core18/1932 -o ro,nodev,suid
systemd+     183  0.0  0.0  28012  5852 ?        Ss   00:07   0:00 /lib/systemd/systemd-networkd
systemd+     185  0.0  0.0  25156  9920 ?        Ss   00:07   0:00 /lib/systemd/systemd-resolved
root         220  0.0  0.0 238208  4744 ?        Ssl  00:07   0:00 /usr/lib/accountsservice/accounts-daemon
root         223  0.0  0.0   8524  1696 ?        Ss   00:07   0:00 /usr/sbin/cron -f
message+     224  0.0  0.0   8256  2884 ?        Ss   00:07   0:00 /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation --syslog-only
root         230  0.0  0.0  29928 14364 ?        Ss   00:07   0:00 /usr/bin/python3 /usr/bin/networkd-dispatcher --run-startup-triggers
syslog       231  0.0  0.0 151608  3084 ?        Ssl  00:07   0:00 /usr/sbin/rsyslogd -n -iNONE
root         234  0.0  0.0 2059004 29060 ?       Ssl  00:07   0:01 /usr/lib/snapd/snapd
root         236  0.0  0.0  17816  5620 ?        Ss   00:07   0:00 /lib/systemd/systemd-logind
daemon       238  0.0  0.0   3776  1540 ?        Ss   00:07   0:00 /usr/sbin/atd -f
root         248  0.0  0.0   7348  1444 pts/0    Ss+  00:07   0:00 /sbin/agetty -o -p -- \u --noclear --keep-baud console 115200,38400,9600 linux
root         250  0.0  0.0  13068  4500 ?        Ss   00:07   0:00 sshd: /usr/sbin/sshd -D [listener] 0 of 10-100 startups
root         260  0.0  0.0 233592  4804 ?        Ssl  00:07   0:00 /usr/libexec/polkitd --no-debug
root         269  0.0  0.0 108196 15804 ?        Ssl  00:07   0:00 /usr/bin/python3 /usr/share/unattended-upgrades/unattended-upgrade-shutdown --wait-for-signal

root@container1:~# exit
exit

Commands when container is accessed with Sudo

$ lxc exec $LXD_CONTAINER_NAME -- sudo --user ubuntu --login
To run a command as administrator (user "root"), use "sudo <command>".
See "man sudo_root" for details.

ubuntu@container1:~$ ls -lah /snap/core18/
ls: cannot access '/snap/core18/1932': Permission denied
total 2.5K
drwxr-xr-x 3 root root 4 Nov  3 11:33 .
drwxr-xr-x 6 root root 7 Nov  3 11:33 ..
d????????? ? ?    ?    ?            ? 1932
lrwxrwxrwx 1 root root 4 Nov  3 11:33 current -> 1932

ubuntu@container1:~$ uname -a
Linux container1 5.8.0-26-generic #27-Ubuntu SMP Wed Oct 21 22:29:16 UTC 2020 x86_64 x86_64 x86_64 GNU/Linux

ubuntu@container1:~$ ps fauxww
USER         PID %CPU %MEM    VSZ   RSS TTY      STAT START   TIME COMMAND
root         680  0.0  0.0  11680  3232 pts/0    Ss   00:08   0:00 sudo --user ubuntu --login
ubuntu       681  0.0  0.0  10056  4180 pts/0    S    00:08   0:00  \_ -bash
ubuntu       698  0.0  0.0  11476  2236 pts/0    R+   00:09   0:00      \_ ps fauxww
root           1  0.0  0.0 170676  8224 ?        Ss   00:07   0:00 /sbin/init
root          62  0.0  0.0  36240  9524 ?        Ss   00:07   0:00 /lib/systemd/systemd-journald
root          94  0.0  0.0  20452  3364 ?        Ss   00:07   0:00 /lib/systemd/systemd-udevd
root         113  0.0  0.0   3692  1236 ?        Ss   00:07   0:00 snapfuse /var/lib/snapd/snaps/lxd_18137.snap /snap/lxd/18137 -o ro,nodev,suid
root         114  0.0  0.0   3624  1188 ?        Ss   00:07   0:00 snapfuse /var/lib/snapd/snaps/snapd_9721.snap /snap/snapd/9721 -o ro,nodev,suid
root         115  0.0  0.0   3732  1412 ?        Ss   00:07   0:00 snapfuse /var/lib/snapd/snaps/core18_1932.snap /snap/core18/1932 -o ro,nodev,suid
systemd+     183  0.0  0.0  28012  5852 ?        Ss   00:07   0:00 /lib/systemd/systemd-networkd
systemd+     185  0.0  0.0  25156  9920 ?        Ss   00:07   0:00 /lib/systemd/systemd-resolved
root         220  0.0  0.0 238208  4744 ?        Ssl  00:07   0:00 /usr/lib/accountsservice/accounts-daemon
root         223  0.0  0.0   8524  1696 ?        Ss   00:07   0:00 /usr/sbin/cron -f
message+     224  0.0  0.0   8256  2884 ?        Ss   00:07   0:00 /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation --syslog-only
root         230  0.0  0.0  29928 14364 ?        Ss   00:07   0:00 /usr/bin/python3 /usr/bin/networkd-dispatcher --run-startup-triggers
syslog       231  0.0  0.0 151608  3084 ?        Ssl  00:07   0:00 /usr/sbin/rsyslogd -n -iNONE
root         234  0.0  0.0 2059004 29060 ?       Ssl  00:07   0:01 /usr/lib/snapd/snapd
root         236  0.0  0.0  17816  5620 ?        Ss   00:07   0:00 /lib/systemd/systemd-logind
daemon       238  0.0  0.0   3776  1540 ?        Ss   00:07   0:00 /usr/sbin/atd -f
root         248  0.0  0.0   7348  1444 pts/0    Ss+  00:07   0:00 /sbin/agetty -o -p -- \u --noclear --keep-baud console 115200,38400,9600 linux
root         250  0.0  0.0  13068  4500 ?        Ss   00:07   0:00 sshd: /usr/sbin/sshd -D [listener] 0 of 10-100 startups
root         260  0.0  0.0 233592  4804 ?        Ssl  00:07   0:00 /usr/libexec/polkitd --no-debug
root         269  0.0  0.0 108196 15804 ?        Ssl  00:07   0:00 /usr/bin/python3 /usr/share/unattended-upgrades/unattended-upgrade-shutdown --wait-for-signal

lxc config show for container

$ lxc config show --expanded $LXD_CONTAINER_NAME
architecture: x86_64
config:
  image.architecture: amd64
  image.description: ubuntu 20.10 amd64 (release) (20201103)
  image.label: release
  image.os: ubuntu
  image.release: groovy
  image.serial: "20201103"
  image.type: squashfs
  image.version: "20.10"
  volatile.base_image: c010961d207c567fdb7cdd72405812d627151140f436835b115f9b673d0c02f6
  volatile.eth0.host_name: veth424c96e7
  volatile.eth0.hwaddr: 00:16:3e:93:2f:34
  volatile.idmap.base: "0"
  volatile.idmap.current: '[{"Isuid":true,"Isgid":false,"Hostid":1000000,"Nsid":0,"Maprange":1000000000},{"Isuid":false,"Isgid":true,"Hostid":1000000,"Nsid":0,"Maprange":1000000000}]'
  volatile.idmap.next: '[{"Isuid":true,"Isgid":false,"Hostid":1000000,"Nsid":0,"Maprange":1000000000},{"Isuid":false,"Isgid":true,"Hostid":1000000,"Nsid":0,"Maprange":1000000000}]'
  volatile.last_state.idmap: '[{"Isuid":true,"Isgid":false,"Hostid":1000000,"Nsid":0,"Maprange":1000000000},{"Isuid":false,"Isgid":true,"Hostid":1000000,"Nsid":0,"Maprange":1000000000}]'
  volatile.last_state.power: RUNNING
devices:
  eth0:
    name: eth0
    network: lxdbr0
    type: nic
  root:
    path: /
    pool: default
    type: disk
ephemeral: false
profiles:
- default
stateful: false
description: ""

quater · November 5, 2020, 1:08am

I am observing this behaviour with LXD, both stable and candidate.

$ sudo snap list lxd
Name  Version  Rev    Tracking          Publisher   Notes
lxd   4.7      18169  latest/candidate  canonical✓  -


$ sudo snap list lxd
Name  Version  Rev    Tracking       Publisher   Notes
lxd   4.7      18137  latest/stable  canonical✓  -

stgraber · November 5, 2020, 1:13am

The issue is snapfuse, it’s not started with allow_other which is required for non-root users to have access to the snap data. This must be a snapd bug as at least here I’m certainly seeing it run with that particular option.

@pstolowski

stgraber@castiana:~/data/code/lxc/lxc-ci/bin (master)$ lxc exec lxd-build bash
root@lxd-build:~# ps fauxww
USER         PID %CPU %MEM    VSZ   RSS TTY      STAT START   TIME COMMAND
root         514  0.0  0.0  23204  2488 pts/0    Ss   01:12   0:00 bash
root         524  0.0  0.0  39100  2172 pts/0    R+   01:12   0:00  \_ ps fauxww
root           1  0.0  0.0 225284  6364 ?        Ss   Nov04   0:00 /sbin/init
root          49  0.0  0.0  78492  7336 ?        Ss   Nov04   0:00 /lib/systemd/systemd-journald
root          55  0.0  0.0  33396  2044 ?        Ss   Nov04   0:00 /lib/systemd/systemd-udevd
root         100  0.0  0.0  18152  1536 ?        Ss   Nov04   0:00 snapfuse /var/lib/snapd/snaps/core_10185.snap /snap/core/10185 -o ro,nodev,allow_other,suid
root         106  0.0  0.0  17908  1088 ?        Ss   Nov04   0:00 snapfuse /var/lib/snapd/snaps/go_6633.snap /snap/go/6633 -o ro,nodev,allow_other,suid
root         110  0.0  0.0  17776   164 ?        Ss   Nov04   0:00 snapfuse /var/lib/snapd/snaps/core18_1932.snap /snap/core18/1932 -o ro,nodev,allow_other,suid
root         113  0.0  0.0  17908   164 ?        Ss   Nov04   0:00 snapfuse /var/lib/snapd/snaps/chromium_1373.snap /snap/chromium/1373 -o ro,nodev,allow_other,suid
root         118  0.0  0.0  17776   160 ?        Ss   Nov04   0:00 snapfuse /var/lib/snapd/snaps/go_6439.snap /snap/go/6439 -o ro,nodev,allow_other,suid
root         119  0.0  0.0  17776   160 ?        Ss   Nov04   0:00 snapfuse /var/lib/snapd/snaps/core_10126.snap /snap/core/10126 -o ro,nodev,allow_other,suid
root         122  0.0  0.0  17908  1116 ?        Ss   Nov04   0:00 snapfuse /var/lib/snapd/snaps/gtk-common-themes_1506.snap /snap/gtk-common-themes/1506 -o ro,nodev,allow_other,suid
systemd+     126  0.0  0.0  80096  3652 ?        Ss   Nov04   0:00 /lib/systemd/systemd-networkd
systemd+     165  0.0  0.0  70676  3696 ?        Ss   Nov04   0:00 /lib/systemd/systemd-resolved
root         228  0.0  0.0 170384 13100 ?        Ssl  Nov04   0:00 /usr/bin/python3 /usr/bin/networkd-dispatcher --run-startup-triggers
syslog       229  0.0  0.0 193412  2416 ?        Ssl  Nov04   0:00 /usr/sbin/rsyslogd -n
root         230  0.0  0.0  31304  1456 ?        Ss   Nov04   0:00 /usr/sbin/cron -f
root         231  0.0  0.0  62064  3556 ?        Ss   Nov04   0:00 /lib/systemd/systemd-logind
root         232  0.0  0.1 1309384 24764 ?       Ssl  Nov04   0:11 /usr/lib/snapd/snapd
message+     233  0.0  0.0  49940  2444 ?        Ss   Nov04   0:00 /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation --syslog-only
root         237  0.0  0.0  15968  1056 pts/0    Ss+  Nov04   0:00 /sbin/agetty -o -p -- \u --noclear --keep-baud console 115200,38400,9600 linux
root         239  0.0  0.0 187100 14704 ?        Ssl  Nov04   0:00 /usr/bin/python3 /usr/share/unattended-upgrades/unattended-upgrade-shutdown --wait-for-signal
root         396  0.0  0.0  17968   808 ?        Ss   Nov04   0:00 snapfuse /var/lib/snapd/snaps/chromium_1382.snap /snap/chromium/1382 -o ro,nodev,allow_other,suid
root@lxd-build:~#

As you can see, the allow_other is passed in my case.

pstolowski · November 5, 2020, 2:21pm

Thanks @stgraber! I’ll check this.

pstolowski · November 9, 2020, 3:45pm

I’ve tracked the problem down to snapd-generator for systemd, it’s affecting lxd images (snaps that come preinstalled on the these images) prepared with snap-preseed, the fix is proposed here:

https://github.com/snapcore/snapd/pull/9613

Once landed future groovy images for lxd should not be affected anymore. The problem should also disappear after core18 snap and snapd snap get refreshed, or snapd deb is upgraded inside the container.

quater · November 10, 2020, 11:29pm

@pstolowski Thank you for fixing this!