Rclone: understand basic commands and configuration in command line

[ambiknai]

No description provided.

[ambiknai]

Follow
https://cloud.ibm.com/docs/cloud-object-storage?topic=cloud-object-storage-rclone

Name                 Type
====                 ====
testbucket           s3

e) Edit existing remote
n) New remote
d) Delete remote
r) Rename remote
c) Copy remote
s) Set configuration password
q) Quit config
e/n/d/r/c/s/q> q
root@devmachine1:~# rclone mkdir testbucket:rclonetest
2021/06/15 01:42:17 ERROR : Attempt 1/3 failed with 1 errors and: InvalidGrant: Invalid Grant
	status code: 400, request id: a2027259-b8a0-4b50-957e-b1f0d2255de6, host id: 
2021/06/15 01:42:18 ERROR : Attempt 2/3 failed with 1 errors and: InvalidGrant: Invalid Grant
	status code: 400, request id: d8761daa-2bb6-43db-8c14-d4f6f27317f2, host id: 
2021/06/15 01:42:18 ERROR : Attempt 3/3 failed with 1 errors and: InvalidGrant: Invalid Grant
	status code: 400, request id: c29edde4-5a9a-401a-a478-5e0a0ecf5d0d, host id: 
2021/06/15 01:42:18 Failed to mkdir: InvalidGrant: Invalid Grant
	status code: 400, request id: c29edde4-5a9a-401a-a478-5e0a0ecf5d0d, host id: 
root@devmachine1:~# vi /root/.config/rclone/rclone.conf
root@devmachine1:~# rclone mkdir testbucket:rclonetest
root@devmachine1:~# rclone lsd testbucket
2021/06/15 01:45:29 ERROR : : error listing: directory not found
2021/06/15 01:45:29 Failed to lsd with 2 errors: last error was: directory not found
root@devmachine1:~# rclone ls testbucket
2021/06/15 01:45:41 ERROR : : error listing: directory not found
2021/06/15 01:45:41 Failed to ls with 2 errors: last error was: directory not found
root@devmachine1:~# rclone ls testbucket:bucket
2021/06/15 01:45:49 Failed to ls: AccessDenied: Access Denied
	status code: 403, request id: 439fa2b8-c0c8-4618-998b-ff8b5c585ebf, host id: 
root@devmachine1:~# rclone copy testMakefile testbucket:rclonetest
root@devmachine1:~# 

[ambiknai]

[YiannisGkoufas]

Hi @ambiknai adding to your test, this is the way to mount:

mkdir ~/mount
export AWS_ACCESS_KEY_ID=XXXXXXXX
export AWS_SECRET_ACCESS_KEY=XXXXXXXXXXXXXXXX
rclone mount :s3:yiannis-tpcds-100/ ~/mount --s3-provider=AWS --s3-env-auth=true --s3-endpoint=http://s3.eu.cloud-object-storage.appdomain.cloud --allow-other --vfs-cache-mode=writes --daemon

[ambiknai]

root@devmachine1:~# rclone mount :s3:rclonetest/ ~/mount --s3-provider=IBMCOS --s3-env-auth=true --s3-endpoint=http://s3.us.cloud-object-storage.appdomain.cloud --allow-other --vfs-cache-mode=writes --daemon
root@devmachine1:~# ls ~/mount
testMakefile
root@devmachine1:~# 

[YiannisGkoufas]

that's great @ambiknai, you used env variables access key and secret access key?

[ambiknai]

Yes , I set them as below vars
AWS_ACCESS_KEY_ID and
AWS_SECRET_ACCESS_KEY

[ambiknai]

Checking non root user

Simple rclone scenario

james@devmachine1:~/mount$ rclone copy new.txt testbucket:rclonetest
james@devmachine1:~/mount$ ls -al
total 4
drwxrwxr-x 2 james james  21 Jun 16 03:09 .
drwxr-xr-x 5 james james 136 Jun 16 03:24 ..
-rw-rw-r-- 1 james james   8 Jun 16 03:09 new.txt
james@devmachine1:~/mount$ cat ~/mount/new.txt 
newfile
james@devmachine1:~/mount$ 

mounting as root

root@devmachine1:~# ls -al ~/mount
total 15
drwxr-xr-x  1 root root     0 Jun 16 02:55 .
drwx------ 19 root root  4096 Jun 16 03:29 ..
-rw-r--r--  1 root root     8 Jun 16 03:09 new.txt
-rw-r--r--  1 root root 10557 May 30 23:46 testMakefile
root@devmachine1:~# cat ~/mount/new.txt
newfile
root@devmachine1:~# 

[ambiknai]

rclone mount :s3:rclonetest/ ~/mount --s3-provider=IBMCOS --s3-env-auth=true --s3-endpoint=http://s3.us.cloud-object-storage.appdomain.cloud   --daemon
james@devmachine1:~$ mount
sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime)
proc on /proc type proc (rw,nosuid,nodev,noexec,relatime)
udev on /dev type devtmpfs (rw,nosuid,relatime,size=8180640k,nr_inodes=2045160,mode=755)
devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000)
tmpfs on /run type tmpfs (rw,nosuid,noexec,relatime,size=1642448k,mode=755)
/dev/mapper/ubuntu--vg-root on / type xfs (rw,relatime,attr2,inode64,noquota)
securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime)
tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev)
tmpfs on /run/lock type tmpfs (rw,nosuid,nodev,noexec,relatime,size=5120k)
tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755)
cgroup on /sys/fs/cgroup/unified type cgroup2 (rw,nosuid,nodev,noexec,relatime)
cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,name=systemd)
pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime)
cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_cls,net_prio)
cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory)
cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices)
cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpu,cpuacct)
cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids)
cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb)
cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event)
cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio)
cgroup on /sys/fs/cgroup/rdma type cgroup (rw,nosuid,nodev,noexec,relatime,rdma)
cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset)
cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer)
mqueue on /dev/mqueue type mqueue (rw,relatime)
debugfs on /sys/kernel/debug type debugfs (rw,relatime)
systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=35,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=19767)
hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime,pagesize=2M)
configfs on /sys/kernel/config type configfs (rw,relatime)
fusectl on /sys/fs/fuse/connections type fusectl (rw,relatime)
lxcfs on /var/lib/lxcfs type fuse.lxcfs (rw,nosuid,nodev,relatime,user_id=0,group_id=0,allow_other)
tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=1642444k,mode=700)
s3fs on /tmp/mount type fuse.s3fs (rw,nosuid,nodev,relatime,user_id=0,group_id=0,allow_other)
:s3{9BYaW}:rclonetest on /home/james/mount type fuse.rclone (rw,nosuid,nodev,relatime,user_id=1001,group_id=1001)
james@devmachine1:~$ ls  /home/james/mount
new.txt  testMakefile
james@devmachine1:~$ ls -al rclone mount :s3:rclonetest/ ~/mount --s3-provider=IBMCOS --s3-env-auth=true --s3-endpoint=http://s3.us.cloud-object-storage.appdomain.cloud   --daemon^C
james@devmachine1:~$ ls -al /home/james/mount
total 11
drwxrwxr-x 1 james james     0 Jun 16 04:13 .
drwxr-xr-x 5 james james   156 Jun 16 03:49 ..
-rw-rw-r-- 1 james james    14 Jun 16 03:58 new.txt
-rw-rw-r-- 1 james james 10557 May 30 23:46 testMakefile
james@devmachine1:~$ 

root@devmachine1:~# echo "rootfile" > /tmp/mount/rootuserfile
root@devmachine1:~# ls -al /tmp/mount
total 16
drwxrwxrwx  1 root root     0 Dec 31  1969 .
drwxrwxrwt 12 root root  4096 Jun 16 04:03 ..
----------  1 root root    14 Jun 16 03:58 new.txt
-rw-r--r--  1 root root     9 Jun 16 04:16 rootuserfile
----------  1 root root 10557 Dec 31  1969 testMakefile



james@devmachine1:~$ ls -al /home/james/mount
total 11
drwxrwxr-x 1 james james     0 Jun 16 04:13 .
drwxr-xr-x 5 james james   156 Jun 16 03:49 ..
-rw-rw-r-- 1 james james    14 Jun 16 03:58 new.txt
-rw-rw-r-- 1 james james 10557 May 30 23:46 testMakefile
james@devmachine1:~$ 

root@devmachine1:~# umount /home/james/mount
root@devmachine1:~# mount

mount again with rclone

james@devmachine1:~$ rclone mount :s3:rclonetest/ ~/mount --s3-provider=IBMCOS --s3-env-auth=true --s3-endpoint=http://s3.us.cloud-object-storage.appdomain.cloud   --daemon
james@devmachine1:~$ ls ~/mount
new.txt  rootuserfile  testMakefile
james@devmachine1:~$ ls -al ~/mount
total 12
drwxrwxr-x 1 james james     0 Jun 16 04:17 .
drwxr-xr-x 5 james james   156 Jun 16 03:49 ..
-rw-rw-r-- 1 james james    14 Jun 16 03:58 new.txt
-rw-rw-r-- 1 james james     9 Jun 16 04:16 rootuserfile
-rw-rw-r-- 1 james james 10557 May 30 23:46 testMakefile
james@devmachine1:~$ 

[ambiknai]

tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=1642444k,mode=700)
s3fs on /tmp/mount type fuse.s3fs (rw,nosuid,nodev,relatime,user_id=0,group_id=0,allow_other)
:s3{9BYaW}:rclonetest on /home/james/mount type fuse.rclone (rw,nosuid,nodev,relatime,user_id=1001,group_id=1001)
root@devmachine1:~# umount /home/james/mount
root@devmachine1:~# 

root@dev021:~# rclone mount :s3:rclonetest/ /tmp/mount --s3-provider=IBMCOS --s3-env-auth=true --s3-endpoint=http://s3.us.cloud-object-storage.appdomain.cloud   --daemon --allow-other --vfs-cache-mode=writes 
root@dev021:~# mount
sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime)
proc on /proc type proc (rw,nosuid,nodev,noexec,relatime)
udev on /dev type devtmpfs (rw,nosuid,relatime,size=8180296k,nr_inodes=2045074,mode=755)
devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000)
tmpfs on /run type tmpfs (rw,nosuid,noexec,relatime,size=1642448k,mode=755)
/dev/mapper/ubuntu--vg-root on / type xfs (rw,relatime,attr2,inode64,noquota)
securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime)
tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev)
tmpfs on /run/lock type tmpfs (rw,nosuid,nodev,noexec,relatime,size=5120k)
tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755)
cgroup on /sys/fs/cgroup/unified type cgroup2 (rw,nosuid,nodev,noexec,relatime)
cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,name=systemd)
pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime)
cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_cls,net_prio)
cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpu,cpuacct)
cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory)
cgroup on /sys/fs/cgroup/rdma type cgroup (rw,nosuid,nodev,noexec,relatime,rdma)
cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices)
cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids)
cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio)
cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event)
cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer)
cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb)
cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset)
mqueue on /dev/mqueue type mqueue (rw,relatime)
systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=41,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12655)
hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime,pagesize=2M)
debugfs on /sys/kernel/debug type debugfs (rw,relatime)
configfs on /sys/kernel/config type configfs (rw,relatime)
fusectl on /sys/fs/fuse/connections type fusectl (rw,relatime)
lxcfs on /var/lib/lxcfs type fuse.lxcfs (rw,nosuid,nodev,relatime,user_id=0,group_id=0,allow_other)
tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=1642444k,mode=700)
:s3{9BYaW}:rclonetest on /tmp/mount type fuse.rclone (rw,nosuid,nodev,relatime,user_id=0,group_id=0,allow_other)
root@dev021:~# ls /tmp/mount
ls: reading directory '/tmp/mount': Software caused connection abort

root@devmachine1:~# umount /home/james/mount
root@devmachine1:~# 

root@dev021:~# rclone mount :s3:rclonetest/ /tmp/mount --s3-provider=IBMCOS --s3-env-auth=true --s3-endpoint=http://s3.us.cloud-object-storage.appdomain.cloud   --daemon --allow-other --vfs-cache-mode=writes  -vv --log-file rclone.log 
root@dev021:~# ls /tmp/mount
new.txt  rootuserfile  testMakefile
root@dev021:~#