#Default armbian login: root/1234 ######## references #https://nasilemaktech.com/guide-lizardfs-drive-level-redundancy/ #https://marc.xn--wckerlin-0za.ch/computer/run-a-stable-lizardfs # # wget -O - http://packages.lizardfs.com/lizardfs.key | apt-key add - ##For Debian do: ## WARNING : AMD64 only !!! NO arm (yet) # apt-get install lsb-release # echo "deb http://packages.lizardfs.com/debian/$(lsb_release -sc) $(lsb_release -sc) main" > /etc/apt/sources.list.d/lizardfs.list # echo "deb-src http://packages.lizardfs.com/debian/$(lsb_release -sc) $(lsb_release -sc) main" >> /etc/apt/sources.list.d/lizardfs.list ------------------------------- update firmware usb <-> sata bridge # cd /tmp scp dth@filer5:/export/software/linux/odroid/hc2/jms578fwupdater.tgz . tar xf jms578fwupdater.tgz cd JMS578FwUpdater/ # Show version information of bridge chip firmware ./JMS578FwUpdate -d /dev/sda -v disable spin down /tmp/JMS578FwUpdater# ./JMS578FwUpdate -d /dev/sda -f ./JMS578_Hardkernel_v173.01.00.01.bin -b ./backup.bin -t 0 Update Firmware file name: ./JMS578_Hardkernel_v173.01.00.01.bin Backup Firmware file name: ./backup.bin Auto spin-down timer: 0 min. Backup the ROM code sucessfully. Programming & Compare Success!! ---------------------- Setup: cs1 = chunk server 1 & master server 1 cs2 = chunk server 2 & master server 2 (shadow) ---------------------- ---------------------- as user dth vi ~/.tmux.conf # Rebind to Ctrl-a set -g prefix C-a unbind C-b bind a send-prefix bind C-a last-window bind '"' choose-window # Rebind pane splitting unbind % bind - split-window -v bind _ split-window -h # Set status bar set -g status-bg colour236 set -g status-fg white set -g status-left '#[fg=green]#H' # Highlight active window set-window-option -g window-status-current-bg colour164 # Force 256 colors set -g default-terminal "screen-256color" ------------------------------- # set auto login ssh public key mkdir ~/.ssh echo "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCydiLwWmO+qfZ7o9jVC6PlosuZUAcQzQwLhlVUbrSSPQV6Mww7VOa4nV+vufHe6R9bcWq/TntE9oNPpmqfXcHc2xIevVWsAroBWEzQz54YHDkWRpq3aSBhoDn0DBYVT83qnE8hQc6btmz9g3h7lXvSZo7gIpDKwMT3RFSENYbRFzzXDkwhTNMDB0fGonm2O4vwscC9JBD3+mmk5w/F/Q4MHHw9wh4MWFsniIem5f8lnhMoZOyAVInbLnSxxFoO8VRY4t7ZDOUK6m40q9gAlVBSOZ405VOLJbYOq8gaZ0DfGA9OITy45W8lNlvaGOYtkW0O3YHBRL4yi5c8SOO7GITwOQEEHLFbHBSr9x1/ue091NNpb/m5AmVEnRrBNYRZPx1tLKUmGieoZHTyLd2P7PTFMRp20s/DFoyRRVchyptNYsgEjENeQvl0scpbiZKJY5QsfjP+xiVqgZcotd4NxFwJodGQ5L8hY0l6aHTqLFnvfiMN2b1bETD7WDqOW1S7xxfQAfBWJW8lEnaVoXxUVxDdVnO7JGYly3105eVm+MLadpTgdwpkchmH19kliFMtHlPxvbeHaLkliAIBd/GFc2WjSsGvghhh6s2atcpXZMuSu0al0YDxGfuTA3CGs6V4Cw7hYr3wok+G9vtFuztrTwVDZ+5xFtlL4X2ALpdM6vJDyw== dth@imac27" >> ~/.ssh/authorized_keys ------------------------------- #use cache to speedup download cat <> /etc/apt/apt.conf Acquire::http::Proxy "http://cache:3142/"; Acquire::https::Proxy "DIRECT"; EOF #change /etc/apt/sources.list deb http://deb.debian.org/debian stretch main contrib non-free #deb-src http://deb.debian.org/debian stretch main contrib non-free deb http://deb.debian.org/debian stretch-updates main contrib non-free #deb-src http://deb.debian.org/debian stretch-updates main contrib non-free deb http://deb.debian.org/debian stretch-backports main contrib non-free #deb-src http://deb.debian.org/debian stretch-backports main contrib non-free deb http://security.debian.org/ stretch/updates main contrib non-free #deb-src http://security.debian.org/ stretch/updates main contrib non-free #and to #/etc/apt/sources.list.d/armbian.list deb http://apt.armbian.com stretch main stretch-utils stretch-desktop ---------------------- # install HDD shutdown script wget https://dn.odroid.com/5422/script/odroid.shutdown install -o root -g root -m 0755 ./odroid.shutdown /lib/systemd/system-shutdown/odroid.shutdown -------------------------- add this to /etc/hosts on every machine: cat <> /etc/hosts 192.168.42.139 ml1 192.168.42.140 ms1 mfsmaster 192.168.42.141 cs1 192.168.42.142 cs2 192.168.42.143 cs3 192.168.42.144 cs4 192.168.42.145 cs5 192.168.42.146 cs6 192.168.42.147 cs7 192.168.42.15 cache EOF apt update apt -y dist-upgrade apt install xfsprogs procinfo powertop mosh tmux gdisk ############ CHUNK SERVER ######################### apt install lizardfs-chunkserver #enable to auto start it at bootup: systemctl enable lizardfs-chunkserver use gdisk to partition hd mkfs.xfs /dev/sda1 XFS enables write barriers by default since kernel version 2.6.17, but they can be disabled using explicit nobarrier. As you can see in your mount output, nobarrier option is not used. According to XFS FAQ, if write barriers are disabled without the explicit mount option, a log entry is generated. You can check the kernel logs on running system with dmesg. Kernels after version 4.10 will always perform integrity operations and barrier/nobarrier mount option is ignored (as documented in man 5 xfs). #make mount point mkdir /mnt/lizardfs use blkid to make fstab entry: eg: vi /etc/fstab #execute this command: :r!blkid |grep sda1 and make it like : UUID=254e6a97-b600-48e1-84ec-ad8cd4778850 /mnt/lizardfs xfs rw,noexec,nodev,noatime,nodiratime,largeio,inode64 0 1 move config and other files to the new version, for future upgrade cd /etc mv lizardfs mfs; ln -s mfs lizardfs cd /var/lib/ mv lizardfs mfs; ln -s mfs lizardfs echo 'WORKING_USER = root' >> /etc/mfs/mfschunkserver.cfg echo 'WORKING_GROUP = root' >> /etc/mfs/mfschunkserver.cfg echo '/mnt/lizardfs' >> /etc/mfs/mfshdd.cfg ###################### MASTER SERVER ############# apt-get install lizardfs-master cd /etc mv lizardfs mfs; ln -s mfs lizardfs cd /var/lib/ mv lizardfs mfs; ln -s mfs lizardfs echo 'PERSONALITY = master' >> /etc/mfs/mfsmaster.cfg echo 'WORKING_USER = root' >> /etc/mfs/mfsmaster.cfg echo 'WORKING_GROUP = root' >> /etc/mfs/mfsmaster.cfg touch /var/lib/mfs/metadata.mfs echo '192.168.42.0/24 / rw,alldirs,maproot=0' >> /etc/mfs/mfsexports.cfg echo 'LIZARDFSMASTER_ENABLE=true' >> /etc/default/lizardfs-master systemctl enable lizardfs-master ###################### metalogger SERVER ############# apt-get install lizardfs-metalogger cd /etc mv lizardfs mfs; ln -s mfs lizardfs cd /var/lib/ mv lizardfs mfs; ln -s mfs lizardfs ###################### SERVER ############# ------------------- script to automate making servers: https://gist.github.com/badri/299a536b7291a7bc1c09 --------------------------------------------- source: https://marc.wäckerlin.ch/computer/evaluation-of-distributed-filesystems-for-docker-swarm#Setup LizardFS Following the instructions, I install: universum: master, chunkserver urknall: chunkserver, metalogger pulsar: chunkserver, metalogger raum: chunkserver, cgi-server Master Setup sudo apt-get install lizardfs-master sudo cp /usr/share/doc/lizardfs-master/examples/* /etc/lizardfs echo "192.168.99.0/24 / rw,alldirs,maproot=0" | sudo tee -a /etc/lizardfs/mfsexports.cfg sudo cp /var/lib/lizardfs/metadata.mfs.empty /var/lib/lizardfs/metadata.mfs sudo systemctl enable lizardfs-master sudo systemctl start lizardfs-master Shadow Master MASTER=universum echo "PERSONALITY = shadow" | sudo tee -a mfsmaster.cfg echo "MASTER_HOST = $MASTER" | sudo tee -a mfsmaster.cfg Chunkserver Setup MASTER=universum sudo apt-get install lizardfs-chunkserver sudo mkdir /var/lib/lizardfs/chunk sudo chown lizardfs.lizardfs /var/lib/lizardfs/chunk echo "/var/lib/lizardfs/chunk" | sudo tee /etc/lizardfs/mfshdd.cfg echo "MASTER_HOST = $MASTER" | sudo tee /etc/lizardfs/mfschunkserver.cfg sudo systemctl enable lizardfs-chunkserver sudo systemctl start lizardfs-chunkserver Metalogger Setup MASTER=universum sudo apt-get install lizardfs-metalogger echo "MASTER_HOST = $MASTER" | sudo tee /etc/lizardfs/mfsmetalogger.cfg sudo systemctl enable lizardfs-metalogger sudo systemctl start lizardfs-metalogger CGI Server MASTER=universum sudo apt-get install lizardfs-cgiserv Administration sudo apt-get install lizardfs-adm man lizardfs-admin Client MASTER=universum sudo apt-get install lizardfs-client sudo mkdir /var/lizardfs sudo mfsmount -H $MASTER /var/lizardfs Add mount command to /etc/fstab: echo "mfsmount /var/lizardfs fuse rw,mfsmaster=$MASTER,mfsdelayedinit 0 0" | sudo tee -a /etc/fstab Tests Finished after about 1 day: Copy data from glusterfs (from local filesystem): sudo rsync -avP /var/gluster/volumes/ /var/lizardfs/ Problems I had several problems, but at least got good feedback from the community: lizardfs becomes slow, unstable when growing and with replication Snapshot Usage Consideration temporary metadata file exists Trash Does Not Cleanup So I ended up with the following configuration: Master server configuration in file /etc/mfs/mfsmaster.cfg: LOAD_FACTOR_PENALTY = 0.5 ENDANGERED_CHUNKS_PRIORITY = 0.6 REJECT_OLD_CLIENTS = 1 CHUNKS_WRITE_REP_LIMIT = 20 CHUNKS_READ_REP_LIMIT = 100 Chunk server configuration in file /etc/mfs/mfschunkserver.cfg: MASTER_HOST = universum HDD_TEST_FREQ = 3600 ENABLE_LOAD_FACTOR = 1 NR_OF_NETWORK_WORKERS = 10 NR_OF_HDD_WORKERS_PER_NETWORK_WORKER = 4 PERFORM_FSYNC = 0