pve5

SSD boot drive on sideways tray ( in future move to HDD)


nfs over RDMA
```
[/etc/exports]
/mnt/pokadot 192.168.199.0/24(rw,async,no_subtree_check)
/mnt/media 192.168.199.0/24(rw,async,no_subtree_check)
/mnt/photos 192.168.199.0/24(rw,async,no_subtree_check) 

```


Back Drive Carriages
```
A.300GB HDD (unused, to be used for boot)
B.2TB N300 used 
C.120 GB HP SSD ZFS ssd pool
D.128 GB Crucial M4 ZFS ssd pool
E.128 GB Crucial M4 ZFS ssd pool
F: Empty tray

Room for 3 more sideways ssd trays
```


zfs primer from servethehome


Toshiba N300 (TOP)
/dev/sdb X0S0A002FPBG
```
root@pve5:/proc/fs/nfsd# hdparm -Ttv /dev/disk/by-id/ata-TOSHIBA_HDWG21C_X0S0A002FPBG

/dev/disk/by-id/ata-TOSHIBA_HDWG21C_X0S0A002FPBG:
 multcount     =  0 (off)
 IO_support    =  1 (32-bit)
 readonly      =  0 (off)
 readahead     = 256 (on)
 geometry      = 1458933/255/63, sectors = 23437770752, start = 0
 Timing cached reads:   15420 MB in  1.99 seconds = 7736.45 MB/sec
 Timing buffered disk reads: 718 MB in  3.01 seconds = 238.75 MB/sec
```




```


zpool create pool0a -m /mnt/pool0a -o ashift=12 /dev/disk/by-id/ata-TOSHIBA_HDWG21C_X0S0A002FPBG
#zpool create pool0a/media 
zfs set atime=off pool0a
zfs set compression=lz4 pool0a
zfs get all pool0a
#a dataset is a file system
#create a dataset specifically useful for large media files
zfs create -o mountpoint=/mnt/media -o recordsize=1M pool0a/media
zfs set recordsize=1M
zfs set xattr=sa dnodesize=auto pool0a/media
zfs set acltype=posixacl pool0a/media

#create a dataset for general purpose files like renders and animations and python coding 
zfs create -o mountpoint=/mnt/pokadot pool0a/pokadot
#setup for nfs sharing
zfs set xattr=sa dnodesize=auto pool0a/pokadot
zfs set acltype=posixacl pool0a/pokadot

#create photo
zfs create -o mountpoint=/mnt/photos pool0a/photos
zfs set xattr=sa dnodesize=auto pool0a/photos
zfs set acltype=posixacl pool0a/photos

#create arqvm
root@pve5:/mnt# zfs create -o mountpoint=/mnt/arqvm pool0a/arqvm
root@pve5:/mnt# zfs set xattr=sa dnodesize=auto pool0a/arqvm
root@pve5:/mnt# zfs set acltype=posixacl pool0a/arqvm
```

nfs
```
/etc/modules
svcrdma
apt-get install nfs-kernel-server
echo rdma 20049 > /proc/fs/nfsd/portlist
```

Toshiba N300 (BOTTOM) (removed)
/dev/sdb X0S0A05AFPBG
```
root@pve5:/proc/fs/nfsd# hdparm -Ttv /dev/disk/by-id/ata-TOSHIBA_HDWG21C_X0S0A05AFPBG 

/dev/disk/by-id/ata-TOSHIBA_HDWG21C_X0S0A05AFPBG:
 multcount     =  0 (off)
 IO_support    =  1 (32-bit)
 readonly      =  0 (off)
 readahead     = 256 (on)
 geometry      = 1458933/255/63, sectors = 23437770752, start = 0
 Timing cached reads:   12608 MB in  1.99 seconds = 6321.10 MB/sec
 Timing buffered disk reads: 728 MB in  3.00 seconds = 242.58 MB/sec
```



```
zpool create /dev/disk/by-id/ata-TOSHIBA_HDWG21C_X0S0A05AFPBG
```
Supermicro X9DRD-7LN4F
Rev 1.02


bindmountpoint
add local mount points to container that will share files over smb
```
pct set 5001 -mp3 mp=/mnt/pokadot,/mnt/pokadot
pct set 5001 -mp4 mp=/mnt/media,/mnt/media
pct set 5001 -mp5 mp=/mnt/photos,/mnt/photos
pct set 75005 -mp6 mp=/mnt/arqvm,/mnt/arqvm
```







```
[pokadot]
   comment = pokadot files zfs recordsize=128k
   path = /mnt/pokadot
   browseable = yes
   valid users = cupcake
   writable = yes
   read only = no
   create mask = 0664
   directory mask = 2775

[media]
   comment = media files zfs recordsize=1M
   path = /mnt/media
   browseable = yes
   valid users = cupcake
   writable = yes
   read only = no
   create mask = 0664
   directory mask = 2775

[photos]
   comment = photos files zfs recordsize=128
   path = /mnt/photos
   browseable = yes
   valid users = cupcake
   writable = yes
   read only = no
   create mask = 0664
   directory mask = 2775
```


zfs on 3 ssd disks (no redundancy, if one drive fails , all drives fail which is fine for backuped vms like Toph)
```
0a
zpool create poolssd0a -m /mnt/poolssd0a -o ashift=12 /dev/disk/by-id/ata-HP_SSD_S600_120GB_HBSA18241701420 /dev/disk/by-id/ata-M4-CT128M4SSD2_000000001224090D478B /dev/disk/by-id/ata-M4-CT128M4SSD2_00000000123409140AA7
zfs set atime=off poolssd0a
zfs set compression=lz4 poolssd0a
zfs get all poolssd0a

#zpool create poolssd0a/vm 
#a dataset is a file system
#create a dataset specifically useful for large media files
zfs create -o mountpoint=/mnt/vm -o recordsize=64K poolssd0a/vm
zfs set xattr=sa dnodesize=auto poolssd0a/vm
zfs set acltype=posixacl poolssd0a/vm
0a
```


[port 20049] https://tech-jungle.com/setting-up-proxmox-with-shared-storage-over-infiniband/
```
mkdir /tools

Next, inside the directory we’ve just made, create a script and name it “open_rdma_port.sh” that has the following contents:

#!/bin/sh
# Add the RDMA Port for Infiniband to Portlist
echo rdma 20049 > /proc/fs/nfsd/portlist
exit 0

After saving and closing the file, make it executable:

chmod +x open_rdma_port.sh

And finally, create a cron entry to run this script every time the system boots. Said entry should look as such:
crontab -e
@reboot /tools/open_rdma_port.sh
```

```

root@pve5:~# zfs create -o recordsize=64K pool0a/vm
root@pve5:~# zfs set xattr=sa dnodesize=auto pool0a/vm
root@pve5:~# zfs set acltype=posixacl pool0a/vm
```


OSX

```
wget https://github.com/thenickdude/pve-edk2-firmware/releases/download/2.202002-1-macos/pve-edk2-firmware_2.202002-1_all.deb

dpkg -i pve-edk2-firmware_2.202002-1_all.deb
apt-mark hold pve-edk2-firmware
```


ip addr
24:be:05:ff:ff:cd:8e:32
/etc/udev/rules.d/70-persistent-ipoib.rules
```
ACTION=="add", SUBSYSTEM=="net", DRIVERS=="?*", ATTR{type}=="32", ATTR{address}=="?*24:be:05:ff:ff:cd:8e:32", NAME="ib0a"
```