[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

some problems with pf and carp



hi all,
finally i've managed to get my openbsd firewall with carp (hot failover)
running on two soekris 4081 machines...
nearly everything is working fine ... except some remaining issues
1) As time goes on - nothing special happens - simple both machines are
killing each other machine (and i don't have any idea why). If they then
start both at the same time - then they will also start to kill each
other. It only works if i start one machine - let it run - and then
start the other machine (so - could it be an issue with pfsync?)
2) I have to add a rule for nearly each outside service to get it
working - but i have a generall rule so that all outside services should
work fine. Why is this ?
Attached my current pf.conf and rc
Please have a look at it - and tell me what i am doing wrong
best regards
Wolfgang
# Macros
###########################################
#
# Die einzelnen Schnittstellen
#
ext_if="sis0"
int_if="sis1"
ext_carp="carp0"
int_carp="carp1"
cross_if="sis2"
lo_if="lo0"
# The dialog Private Address Range
prv_ad = "172.16.0.0/24"
# My Primary External Address
ext_ad = "81.223.6.242"
# My Normal Extern Addresses
ext_ads = "81.223.6.244/24"
# Protocols for which we are doing nat
nat_proto = "{tcp, udp, icmp, gre }"
###########################################
#
# Unsere externen IP's
#
ext_webprimary="81.223.6.242"
ext_websecondary="81.223.6.244"
ext_mailserver="81.223.6.253"
ext_nameserver="81.223.6.243"
ext_moneyrunner="81.223.6.247"
ext_sputnik="81.223.6.252"
ext_area51="81.223.6.254"
ext_idrahdurch="81.223.6.249"
ext_openvpn="81.223.6.242"
#
# Dazu die internen
#
int_webprimary="172.16.0.46"
int_websecondary="172.16.0.79"
int_mailserver="172.16.0.46"
int_webproxy="172.16.0.46"
int_nameserver="172.16.0.46"
int_openvpn="172.16.0.48"
lowqueueports = "{ 25, 110, 80, 443, 21}"
highqueueports = "{ 53, 22, 23 }"
torrentports = "6881:6999"
# Reservierte Adressen
prv_ads = "{10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16, 224.0.0.0/4, 240.0.0.0/5, 127.0.0.0/8, 0.0.0.0}"
# Tables
table <DMZ> persist {172.16.0.13, 172.16.0.27, 172.16.0.46, 172.16.0.48, 172.16.0.79}
table <WEBSERVERS> persist {172.16.0.46, 172.16.0.48, 172.16.0.79}
table <MAILSERVERS> persist {172.16.0.46}
#######################################################
#
# Options
#
set timeout             { interval 10, frag 30 }
set timeout             { tcp.first 120, tcp.opening 30, tcp.established 86400 }
set timeout             { tcp.closing 900, tcp.finwait 45, tcp.closed 90 }
set timeout             { udp.first 60, udp.single 30, udp.multiple 60 }
set timeout             { icmp.first 20, icmp.error 10 }
set timeout             { other.first 60, other.single 30, other.multiple 60 }
set timeout             { adaptive.start 0, adaptive.end 0 }
set loginterface $ext_if
# Limit the number of fragments kept in memory to 5000
set limit               { states 50000, frags 5000 }
set optimization  	conservative
set block-policy 	drop
#######################################################
#
# Scrub Rules
#
# Scrub all packetes coming from the world
scrub in on $ext_if from any
#######################################################
#
# Packet Queuing Rules
altq on $ext_if priq bandwidth 5Mb queue{ highqueue, lowqueue, bitqueue, other }
queue highqueue priority 14 priq(red) 
queue lowqueue priority 13 priq(red) 
queue bitqueue priority 12 priq(red) 
queue other priority 11 priq(default) 
#######################################################
#
# Packet Redirection rules
# enable the ftp-proxy
rdr on $int_if proto tcp from any to any port 21 -> 127.0.0.1 port 8081
rdr on $int_if proto tcp from any to any port 20 -> 127.0.0.1 port 8081
# Do Simple Masquerading
nat on $ext_if inet proto $nat_proto from $prv_ad to any -> $ext_ad
# Redirect Packets
rdr on {$ext_if, $int_if} inet proto tcp \
	from any to $ext_webprimary port 80 -> $int_webprimary port 80
rdr on {$ext_if, $int_if} inet proto tcp \
	from any to $ext_websecondary port 80 -> $int_websecondary port 80
rdr on {$ext_if, $int_if} inet proto tcp \
        from any to $ext_sputnik port 80 -> $int_websecondary port 80
rdr on {$ext_if, $int_if} inet proto tcp \
        from any to $ext_area51 port 80 -> $int_websecondary port 80
rdr on {$ext_if, $int_if} inet proto tcp \
        from any to $ext_moneyrunner port 80 -> $int_websecondary port 80
                                                                                                                                                     
rdr on {$ext_if, $int_if} inet proto tcp \
        from any to $ext_idrahdurch port 80 -> $int_websecondary port 80
rdr on {$ext_if, $int_if} inet proto tcp \
	from any to $ext_mailserver port 25 -> $int_mailserver port 25
rdr on {$ext_if, $int_if} inet proto tcp \
	from any to $ext_mailserver port 110 -> $int_mailserver port 110
rdr on {$ext_if, $int_if} inet proto {tcp, udp} \
	from any to $ext_nameserver port 53 -> $int_nameserver port 53
rdr on $ext_if inet proto udp \
	from any to $ext_openvpn port 5000:5010 -> $int_openvpn
 
# Damit unsere eigenen externen Adressen intern auch erreichbar sind
nat on $int_if proto tcp from $prv_ad to $int_webprimary port 80 -> ($int_carp)
nat on $int_if proto tcp from $prv_ad to $int_websecondary port 80 -> ($int_carp)
nat on $int_if proto {tcp, udp} from $prv_ad to $int_nameserver port 53 -> ($int_carp)
nat on $int_if proto tcp from $prv_ad to $int_mailserver port {25, 110} -> ($int_carp)
# Send all outgoing traffic (expect the traffic from the proxy) on port 80 to my web proxy
#no rdr on $int_if inet proto tcp \
#	from $int_webproxy to any port 80
#rdr on $int_if proto tcp from $prv_ad to any port 80 -> \
#   $int_webproxy port 3128
#no nat on $int_if proto tcp from $int_if to $prv_ad
#nat on $int_if proto tcp from $prv_ad to $int_webproxy port 3128 -> \
#   ($int_carp)
#######################################################
#
# Packet Filtering Rules
# generally block all incoming packets on the external interface
block in log on $ext_if all
block out log on $ext_if all
# pass all to loopback interface
pass quick on lo0 all
# let pfsync and carp traffic through
pass quick on { $cross_if } proto pfsync
pass quick on { $ext_if $int_if } proto carp keep state
# To get the polen vpn tunnel working
pass out quick on $ext_if inet proto gre from any to any
pass out quick on $ext_if inet proto tcp from any to any port pptp
# It seems that the worldcom xtraconnect programm is doing something nifty with its tcp packets
# so we need this extra rule to get it working
pass out quick on $ext_if inet proto tcp from any to any port 9003
# icq
pass out quick on $ext_if inet proto tcp from any to any port 5190
# rsync
pass out quick on $ext_if inet proto tcp from any to any port rsync
# telebanking
pass out quick on $ext_if inet proto tcp from any to any port 3048
# ntp
pass out quick on $ext_if inet proto udp from any to any port ntp
# whois
pass out quick on $ext_if inet proto tcp from any to any port whois
# pass all connections from our lan
pass in quick on $int_if from any to any flags S/SA
pass out quick on $int_if from any to any flags S/SA
# pass all for the webserver
pass in on $ext_if inet proto tcp from any to $int_webprimary port 80 flags S/SA keep state
pass in on $ext_if inet proto tcp from any to $int_websecondary port 80 flags S/SA keep state
# pass all for the mail server
pass in on $ext_if inet proto tcp from any to $int_mailserver port 25 flags S/SA keep state
pass in on $ext_if inet proto tcp from any to $int_mailserver port 110 flags S/SA keep state
# pass all for the name server
pass in on $ext_if inet proto tcp from any to $int_nameserver port 53 flags S/SA keep state
pass in on $ext_if inet proto udp from any to $int_nameserver port 53 keep state
# pass in for openvpn
pass in on $ext_if inet proto udp from any to $int_openvpn port 5000:5010 keep state
# return destination-unreachable to auth requests
block return-icmp in quick on $ext_if proto tcp from any to $ext_ads port auth
# block spoofed packets
block in quick log on $ext_if from $prv_ads
antispoof for $ext_if
antispoof for $int_if
# allow the ping (echo 8)
pass in quick inet proto icmp icmp-type 8 code 0 keep state
pass out quick inet proto icmp icmp-type 8 code 0 keep state
# allow ssh
pass in quick inet proto tcp from any to $ext_ad port 22 flags S/SA keep state
# To Handle the queueing thing
# highqueue
pass out quick on $ext_if inet proto udp from any to any \
	port $highqueueports keep state queue highqueue
pass out quick on $ext_if inet proto tcp from any to any \
	port $highqueueports keep state queue highqueue
# lowqueue
pass out quick on $ext_if inet proto udp from any to any \
	port $lowqueueports keep state queue lowqueue
pass out quick on $ext_if inet proto tcp from any to any \
	port $lowqueueports keep state queue lowqueue
# bitqueue
pass out quick on $ext_if inet proto tcp from any to any \
	port $torrentports keep state queue bitqueue
# pass all connections originating from the firewall
pass out quick on $ext_if inet proto tcp from ($ext_if) to any \
	modulate state queue other
pass out quick on $ext_if inet proto udp from ($ext_if) to any \
	keep state queue other
pass out quick on $ext_if inet proto icmp from ($ext_if) to any
# Example config: Soekris net4xxx Boot for router with DHCP, NAT, VLAN
#
# [email protected]
stty status '^T'
# Set shell to ignore SIGINT (2), but not children;
# shell catches SIGQUIT (3) and returns to single user.
trap : 2
trap : 3
HOME=/; export HOME
PATH=/sbin:/bin:/usr/sbin:/usr/bin
export PATH
if [ "$1" == "shutdown" ]; then
	mount -o ro /
	exit 0
fi
# Filesystem should never be dirty unless we acked while fs was mounted
# read/write
fsck -p
echo mfs: mounting /tmp...
mount_mfs -s 16384 /dev/wd0b /tmp
echo mfs: populating /tmp...
# flashdist makes /var a link to /tmp/var
mkdir /tmp/images
mkdir /tmp/var
mkdir /tmp/var/tmp
mkdir /tmp/var/tmp/vi.recover
mkdir /tmp/var/run
mkdir /tmp/var/log
mkdir /tmp/var/db
mkdir /tmp/var/empty
if [ -d /root ]; then
 cp -R /root /tmp/root
fi
chmod -R 755 /tmp/var
chmod a+rwxt /tmp/var/tmp/vi.recover
touch /tmp/var/run/utmp
touch /tmp/var/log/authlog
touch /tmp/var/log/messages
# Copy over devices created from flashdist into a place where the permissions
# can be changed.  Flashdist already created links to /var/run/dev/XXX for
# these devices.
mkdir /var/run/dev
tar cf - -C /dev/devtmp . | tar xpf - -C /var/run/dev
# You don't need to make databases, but they help ps and some other
# programs ... (skipped kvm for now since /dev/ksyms is a waste of time
# on an embedded router)
#
echo -n "databases:"
echo -n " dev"
dev_mkdb
echo
# Init will do this for us, but to be proper we should do it now, before
# remote login services start
echo -n "securelevel: "
sysctl -w kern.securelevel=1
echo -n "setting carp preempt: "
sysctl -w net.inet.carp.preempt=1
                                                                                                                                   
echo -n "deactivating ARP balancing: "
sysctl -w net.inet.carp.arpbalance=0
echo -n "turning on carp logging: "
sysctl -w net.inet.carp.log=1
echo -n "on panic reboot: "
sysctl -w ddb.panic=0
echo -n "allow gre traffic through the firewall: "
sysctl -w net.inet.gre.allow=1
 
echo -n "watchdog: "
sysctl -w kern.watchdog.period=32
echo -n "watchdog: "
sysctl -w kern.watchdog.auto=1
if [ -f /etc/nshrc -a -x /bin/nsh ]; then
 echo nsh: starting nsh
 nsh -i /etc/nshrc
else
 # Setup hostname, IPs, and pf/nat
 hostname=firewall
 echo hostname: setting hostname to $hostname...
 hostname $hostname
 echo inet: configuring IP on system interfaces...
 ifconfig lo0 127.0.0.1 netmask 255.0.0.0
 ifconfig sis0 81.223.6.250 netmask 255.255.255.240
 ifconfig sis1 172.16.0.254 netmask 255.255.255.0 broadcast 172.16.0.255
 ifconfig sis2 192.168.254.254 netmask 255.255.255.0 broadcast 192.168.254.255
 ifconfig carp0 81.223.6.242 netmask 255.255.255.240 broadcast 81.223.6.255 vhid 1 advskew 100 pass pass1
 ifconfig carp0 alias 81.223.6.243 netmask 255.255.255.240
 ifconfig carp0 alias 81.223.6.244 netmask 255.255.255.240
 ifconfig carp0 alias 81.223.6.245 netmask 255.255.255.240
 ifconfig carp0 alias 81.223.6.246 netmask 255.255.255.240
 ifconfig carp0 alias 81.223.6.247 netmask 255.255.255.240
 ifconfig carp0 alias 81.223.6.248 netmask 255.255.255.240
 ifconfig carp0 alias 81.223.6.249 netmask 255.255.255.240
 ifconfig carp0 alias 81.223.6.252 netmask 255.255.255.240
 ifconfig carp0 alias 81.223.6.253 netmask 255.255.255.240
 ifconfig carp0 alias 81.223.6.254 netmask 255.255.255.240
 ifconfig carp1 172.16.0.2 netmask 255.255.255.0 broadcast 172.16.0.255 vhid 2 pass pass2 
 ifconfig pfsync0 up syncif sis2
 ifconfig pflog0 up
 echo route: adding default route...
 route add default 81.223.6.241 
 echo pf/nat: configuring and enabling...
 pfctl -e -f /etc/pf.conf
fi
if [ -f /etc/syslog.conf ]; then
  echo syslogd: starting log daemon...
  syslogd -p /var/run/log
fi
#echo dhcp: starting server...
#touch /var/db/dhcpd.leases
#dhcpd -q vlan0
if [ -d /etc/ssh -a ! -f /etc/ssh/ssh_host_dsa_key ]; then
 echo -n "ssh-keygen: generating new DSA host key... "
 mount -o rw /dev/wd0a /
 if /usr/bin/ssh-keygen -q -t dsa -f /etc/ssh/ssh_host_dsa_key -N ''; then
  echo done.
 else
  echo failed.
 fi
 mount -o ro /
fi
if [ -d /etc/ssh -a ! -f /etc/ssh/ssh_host_rsa_key ]; then
 echo -n "ssh-keygen: generating new RSA host key... "
 mount -o rw /dev/wd0a /
 if /usr/bin/ssh-keygen -q -t rsa -f /etc/ssh/ssh_host_rsa_key -N ''; then
  echo done.
 else
  echo failed.
 fi
 mount -o ro /
fi
if [ -d /etc/ssh -a ! -f /etc/ssh/ssh_host_key ]; then
 echo -n "ssh-keygen: generating new RSA1 host key... "
 mount -o rw /dev/wd0a /
 if /usr/bin/ssh-keygen -q -t rsa1 -f /etc/ssh/ssh_host_key -N ''; then
  echo done.
 else
  echo failed.
 fi
 mount -o ro /
fi
echo ssh: starting daemon...
sshd 
echo inetd: starting daemon...
inetd
echo cron: starting cron...
mount -o rw /dev/wd0a /
cron
crontab /root/crontab
mount -o ro /