[Gluster-devel] Yet another GlusterFS HA specification

[Sebastien LELIEVRE]

Hi everyone,

Here is a template for a HA storage configuration with GlusterFS.
Configuration is :

2 nodes on a network A (or2 and or3)
1 node on a network B (op1)

We want to provide HA services for both networks A & B

In this example, we also provide a "backup" node for network A

All clients from networks A & B should see the same volume at every time

################ Server 1 ################
volume tbs-clust-op1-local-data
       type storage/posix
       option directory /export/GLUSTER/data
end-volume

volume tbs-clust-op1-local-data-locks
  type features/posix-locks
  subvolumes tbs-clust-op1-local-data
end-volume

volume tbs-clust-op1-local-ns
       type storage/posix
       option directory /export/GLUSTER/namespace
end-volume

volume tbs-clust-or2-remote-data
       type protocol/client
       option transport-type tcp/client
       option remote-host 192.168.28.6
       option remote-subvolume tbs-clust-or2-local-data-locks
end-volume

volume tbs-clust-or3-remote-data
       type protocol/client
       option transport-type tcp/client
       option remote-host 192.168.28.7
       option remote-subvolume tbs-clust-or3-local-data-locks
end-volume

### Performance xlators
volume tbs-clust-or2-iot
       type performance/io-threads
       option thread-count 2
       subvolumes tbs-clust-or2-remote-data
end-volume

volume tbs-clust-or2-wb
       type performance/write-behind
       subvolumes tbs-clust-or2-iot
end-volume

volume tbs-clust-or3-iot
       type performance/io-threads
       option thread-count 2
       subvolumes tbs-clust-or3-remote-data
end-volume

volume tbs-clust-or3-wb
       type performance/write-behind
       subvolumes tbs-clust-or3-iot
end-volume

### Replication on server-side
volume tbs-clust-data-afr
       type cluster/afr
       subvolumes tbs-clust-op1-local-data-locks tbs-clust-or3-wb
tbs-clust-or2-wb
       option replicate *:3
       option self-heal on
end-volume

### Unification on server-side
volume tbs-clust-unify
       type cluster/unify
       subvolumes tbs-clust-data-afr
       option namespace tbs-clust-op1-local-ns
       option scheduler rr
end-volume

### Global Performance xlators
volume tbs-clust-server
       type performance/io-threads
       option thread-count 8
       option cache-size 32MB
       subvolumes tbs-clust-unify
end-volume

### Volumes exportation
volume server
       type protocol/server
       option transport-type tcp/server
       subvolumes tbs-clust-server
       option auth.ip.tbs-clust-op1-local-data-locks.allow
127.0.0.1,10.24.133.178,192.168.28.6,192.168.28.7,192.168.28.5
       option auth.ip.tbs-clust-op1-local-ns.allow 127.0.0.1,10.24.133.178
       option auth.ip.tbs-clust-server.allow
127.0.0.1,10.24.133.178,192.168.28.6,192.168.28.7,192.168.28.5
end-volume


################ Server 2 ################
volume tbs-clust-or2-local-data
       type storage/posix
       option directory /export/GLUSTER/data
end-volume

volume tbs-clust-or2-local-data-locks
  type features/posix-locks
  subvolumes tbs-clust-or2-local-data
end-volume

volume tbs-clust-or2-local-ns
       type storage/posix
       option directory /export/GLUSTER/namespace
end-volume

volume tbs-clust-or3-remote-data
       type protocol/client
       option transport-type tcp/client
       option remote-host 192.168.28.7
       option remote-subvolume tbs-clust-or3-local-data-locks
end-volume

volume tbs-clust-op1-remote-data
       type protocol/client
       option transport-type tcp/client
       option remote-host 10.24.133.178
       option remote-subvolume tbs-clust-op1-local-data-locks
end-volume

### Performance xlators
volume tbs-clust-op1-iot
       type performance/io-threads
       option thread-count 4
       subvolumes tbs-clust-op1-remote-data
end-volume

volume tbs-clust-op1-wb
       type performance/write-behind
       subvolumes tbs-clust-op1-iot
end-volume

### Replication on server-side
volume tbs-clust-data-afr
       type cluster/afr
       subvolumes tbs-clust-or2-local-data-locks
tbs-clust-or3-remote-data tbs-clust-op1-wb
       option replicate *:3
       option self-heal on
end-volume

### Unification on server-side
volume tbs-clust-unify
       type cluster/unify
       subvolumes tbs-clust-data-afr
       option namespace tbs-clust-or2-local-ns
       option scheduler rr
end-volume

### Global Performance xlators
volume tbs-clust-server
       type performance/io-threads
       option thread-count 8
       option cache-size 32MB
       subvolumes tbs-clust-unify
end-volume

### Volumes exportation
volume server
       type protocol/server
       option transport-type tcp/server
       subvolumes tbs-clust-server
       option auth.ip.tbs-clust-or2-local-data-locks.allow
127.0.0.1,192.168.28.6,192.168.28.7,10.24.133.178,192.168.28.5
       option auth.ip.tbs-clust-or2-local-ns.allow 127.0.0.1,192.168.28.6
       option auth.ip.tbs-clust-server.allow
127.0.0.1,192.168.28.6,192.168.28.7,10.24.133.178,192.168.28.5
end-volume


################ Server 3 ################
volume tbs-clust-or3-local-data
       type storage/posix
       option directory /export/GLUSTER/data
end-volume

volume tbs-clust-or3-local-data-locks
  type features/posix-locks
  subvolumes tbs-clust-or3-local-data
end-volume

volume tbs-clust-or3-local-ns
       type storage/posix
       option directory /export/GLUSTER/namespace
end-volume

volume tbs-clust-or2-remote-data
       type protocol/client
       option transport-type tcp/client
       option remote-host 192.168.28.6
       option remote-subvolume tbs-clust-or2-local-data-locks
end-volume

volume tbs-clust-op1-remote-data
       type protocol/client
       option transport-type tcp/client
       option remote-host 10.24.133.178
       option remote-subvolume tbs-clust-op1-local-data-locks
end-volume

### Performance xlators
volume tbs-clust-op1-iot
       type performance/io-threads
       option thread-count 4
       subvolumes tbs-clust-op1-remote-data
end-volume

volume tbs-clust-op1-wb
       type performance/write-behind
       subvolumes tbs-clust-op1-iot
end-volume

### Replication on server-side
volume tbs-clust-data-afr
       type cluster/afr
       subvolumes tbs-clust-or3-local-data-locks
tbs-clust-or2-remote-data tbs-clust-op1-wb
       option replicate *:3
       option self-heal on
end-volume

### Unification on server-side
volume tbs-clust-unify
       type cluster/unify
       subvolumes tbs-clust-data-afr
       option namespace tbs-clust-or3-local-ns
       option scheduler rr
end-volume

### Global Performance xlators
volume tbs-clust-server
       type performance/io-threads
       option thread-count 8
       option cache-size 32MB
       subvolumes tbs-clust-unify
end-volume

### Volumes exportation
volume server
       type protocol/server
       option transport-type tcp/server
       subvolumes tbs-clust-server
       option auth.ip.tbs-clust-or3-local-data-locks.allow
127.0.0.1,192.168.28.7,192.168.28.6,10.24.133.178,192.168.28.5
       option auth.ip.tbs-clust-or3-local-ns.allow 127.0.0.1,192.168.28.7
       option auth.ip.tbs-clust-server.allow
127.0.0.1,192.168.28.7,192.168.28.6,10.24.133.178,192.168.28.5
end-volume



################ client (1, 2, & 3 and even more !) ################
volume tbs-clust
  type protocol/client
  option transport-type tcp/client            # for TCP/IP transport
  option remote-host 192.168.28.6             # IP address of the remote
brick
#  option remote-host glust-or.pitux.org       # rr DNS resolves
192.168.28.6 &
                                               # 192.168.28.7 &
10.24.133.178 if
                                               # there is a weight-balancing
  option remote-subvolume tbs-clust-server    # name of the remote volume
  option transport-timeout 10
end-volume

### Performance xlators
volume tbs-clust-iothreads
       type performance/io-threads
       option thread-count 8
       option cache-size 32MB
       subvolumes tbs-clust
end-volume

volume tbs-clust-write-behind
  type performance/write-behind
  option aggregate-size 131072          # unit in bytes
subvolumes tbs-clust-iothreads

Enjoy !

Sebastien.