[root@controller2 ~]# tail -f /var/log/keystone/keystone-startup.log
_setup_logging_from_conf(product_name, version)
File "/usr/lib/python2.6/site-packages/keystone/openstack/common/log.py", line 525, in _setup_logging_from_conf
filelog = logging.handlers.WatchedFileHandler(logpath)
File "/usr/lib64/python2.6/logging/handlers.py", line 377, in __init__
logging.FileHandler.__init__(self, filename, mode, encoding, delay)
File "/usr/lib64/python2.6/logging/__init__.py", line 827, in __init__
StreamHandler.__init__(self, self._open())
File "/usr/lib64/python2.6/logging/__init__.py", line 846, in _open
stream = open(self.baseFilename, self.mode)
IOError: [Errno 13] Permission denied: '/var/log/keystone/keystone.log'
chown keystone.keystone /var/log/keystone/keystone.log
Saturday, June 28, 2014
Thursday, June 26, 2014
Virtual Ip With Keepalived as Front end for HAproxy server's
Install Keepalived
Virtual Ip 192.168.216.100
HAproxy Ip 192.168.216.101
1. Install Keepalived package:
On RHEL/CentOS:
$ yum install -y centos-release
$ yum install -y keepalived
$ chkconfig keepalived on
2. Tell kernel to allow binding non-local IP into the hosts and apply the changes:
$ echo "net.ipv4.ip_nonlocal_bind = 1" >> /etc/sysctl.conf
$ sysctl -p
Configure Keepalived and Virtual IP
1. Login into LB1 and add following line into /etc/keepalived/keepalived.conf:
vrrp_script chk_haproxy {
script "killall -0 haproxy" # verify the pid existance
interval 2 # check every 2 seconds
weight 2 # add 2 points of prio if OK
}
vrrp_instance VI_1 {
interface eth2 # interface to monitor
state MASTER
virtual_router_id 51 # Assign one ID for this route
priority 101 # 101 on master, 100 on backup
virtual_ipaddress {
192.168.216.100 # the virtual IP
}
track_script {
chk_haproxy
}
}
2. Login into LB2 and add following line into /etc/keepalived/keepalived.conf:
vrrp_script chk_haproxy {
script "killall -0 haproxy" # verify the pid existance
interval 2 # check every 2 seconds
weight 2 # add 2 points of prio if OK
}
vrrp_instance VI_1 {
interface eth2 # interface to monitor
state MASTER
virtual_router_id 51 # Assign one ID for this route
priority 100 # 101 on master, 100 on backup
virtual_ipaddress {
192.168.216.100 # the virtual IP
}
track_script {
chk_haproxy
}
}
3. Start Keepalived in both nodes:
$ sudo /etc/init.d/keepalived start
4. Verify the Keepalived status. LB1 should hold the VIP and the MASTER state while LB2 should run as BACKUP state without VIP:
LB1 IP:
$ ip a | grep -e inet.*eth2
inet 192.168.216.101/24 brd 192.168.216.255 scope global eth2
inet 192.168.216.100/32 scope global eth2
Virtual Ip 192.168.216.100
HAproxy Ip 192.168.216.101
1. Install Keepalived package:
On RHEL/CentOS:
$ yum install -y centos-release
$ yum install -y keepalived
$ chkconfig keepalived on
2. Tell kernel to allow binding non-local IP into the hosts and apply the changes:
$ echo "net.ipv4.ip_nonlocal_bind = 1" >> /etc/sysctl.conf
$ sysctl -p
Configure Keepalived and Virtual IP
1. Login into LB1 and add following line into /etc/keepalived/keepalived.conf:
vrrp_script chk_haproxy {
script "killall -0 haproxy" # verify the pid existance
interval 2 # check every 2 seconds
weight 2 # add 2 points of prio if OK
}
vrrp_instance VI_1 {
interface eth2 # interface to monitor
state MASTER
virtual_router_id 51 # Assign one ID for this route
priority 101 # 101 on master, 100 on backup
virtual_ipaddress {
192.168.216.100 # the virtual IP
}
track_script {
chk_haproxy
}
}
2. Login into LB2 and add following line into /etc/keepalived/keepalived.conf:
vrrp_script chk_haproxy {
script "killall -0 haproxy" # verify the pid existance
interval 2 # check every 2 seconds
weight 2 # add 2 points of prio if OK
}
vrrp_instance VI_1 {
interface eth2 # interface to monitor
state MASTER
virtual_router_id 51 # Assign one ID for this route
priority 100 # 101 on master, 100 on backup
virtual_ipaddress {
192.168.216.100 # the virtual IP
}
track_script {
chk_haproxy
}
}
3. Start Keepalived in both nodes:
$ sudo /etc/init.d/keepalived start
4. Verify the Keepalived status. LB1 should hold the VIP and the MASTER state while LB2 should run as BACKUP state without VIP:
LB1 IP:
$ ip a | grep -e inet.*eth2
inet 192.168.216.101/24 brd 192.168.216.255 scope global eth2
inet 192.168.216.100/32 scope global eth2
Mysql HA with Haproxy and Master-Master replication
Once Mysql master -master replication is done we set HA with those using HAproxy
http://enekumvenamorublog.wordpress.com/2014/06/25/mysql-replication-master-master/
HAProxy
yum install haproxy
To use Haproxy with MYsql we need to create a user in mysql so that haproxy can access it .
GRANT ALL PRIVILEGES ON *.* TO 'haproxy'@'192.168.216.180' IDENTIFIED BY '';
Sample configuration /etc/haproxy/haproxy.cfg
global
log 127.0.0.1 local0 notice
user haproxy
group haproxy
# turn on stats unix socket
stats socket /var/lib/haproxy/stats mode 777
defaults
log global
retries 2
timeout connect 1000
timeout server 5000
timeout client 5000
listen stats 192.168.255.180:80
mode http
stats enable
stats uri /stats
stats realm HAProxy\ Statistics
stats auth admin:password
listen MYSQL 192.168.255.190:3306
balance source
mode tcp
option mysql-check user haproxy
server controller1 192.168.216.130 check
server controller2 192.168.216.135 check
[root@HAPROXY ~]#
http://enekumvenamorublog.wordpress.com/2014/06/25/mysql-replication-master-master/
HAProxy
yum install haproxy
To use Haproxy with MYsql we need to create a user in mysql so that haproxy can access it .
GRANT ALL PRIVILEGES ON *.* TO 'haproxy'@'192.168.216.180' IDENTIFIED BY '';
Sample configuration /etc/haproxy/haproxy.cfg
global
log 127.0.0.1 local0 notice
user haproxy
group haproxy
# turn on stats unix socket
stats socket /var/lib/haproxy/stats mode 777
defaults
log global
retries 2
timeout connect 1000
timeout server 5000
timeout client 5000
listen stats 192.168.255.180:80
mode http
stats enable
stats uri /stats
stats realm HAProxy\ Statistics
stats auth admin:password
listen MYSQL 192.168.255.190:3306
balance source
mode tcp
option mysql-check user haproxy
server controller1 192.168.216.130 check
server controller2 192.168.216.135 check
[root@HAPROXY ~]#
Wednesday, June 25, 2014
Enable HAProxy logging on Centos
Enable HAProxy logging on CentOS
By default, HAProxy will not log to files unless we make some modifications
1. Create rsyslog configuration file
nano /etc/rsyslog.d/haproxy.conf
Add these lines to the file
# Enable UDP port 514 to listen to incoming log messages from haproxy
$ModLoad imudp
$UDPServerRun 514
$template Haproxy,"%msg%\n"
local0.=info -/var/log/haproxy/haproxy.log;Haproxy
local0.notice -/var/log/haproxy/admin.log;Haproxy
# don't log anywhere else
local0.* ~
Restart rsyslog service
/etc/init.d/rsyslog restart
Ref: http://blog.hintcafe.com/post/33689067443/haproxy-logging-with-rsyslog-on-linux
2. Modify the log rotate config to match the new folder:
nano /etc/logrotate.d/haproxy
Change
/var/log/haproxy.log {
daily
rotate 10
missingok
[...]
to
/var/log/haproxy/*.log {
daily
rotate 10
missingok
[...]
Now we can check if HAProxy logging is working.
tail -f /var/log/haproxy/haproxy.log
===================================================
global to have these messages end up in /var/log/haproxy.log you will need to:
1) configure syslog to accept network log events. This is done by adding the '-r' option to the SYSLOGD_OPTIONS in /etc/sysconfig/syslog
2) configure local2 events to go to the /var/log/haproxy.log file. A line like the following can be added to /etc/sysconfig/syslog
local2.* /var/log/haproxy.log
In haproxy conf file add
log 127.0.0.1 local2 info
By default, HAProxy will not log to files unless we make some modifications
1. Create rsyslog configuration file
nano /etc/rsyslog.d/haproxy.conf
Add these lines to the file
# Enable UDP port 514 to listen to incoming log messages from haproxy
$ModLoad imudp
$UDPServerRun 514
$template Haproxy,"%msg%\n"
local0.=info -/var/log/haproxy/haproxy.log;Haproxy
local0.notice -/var/log/haproxy/admin.log;Haproxy
# don't log anywhere else
local0.* ~
Restart rsyslog service
/etc/init.d/rsyslog restart
Ref: http://blog.hintcafe.com/post/33689067443/haproxy-logging-with-rsyslog-on-linux
2. Modify the log rotate config to match the new folder:
nano /etc/logrotate.d/haproxy
Change
/var/log/haproxy.log {
daily
rotate 10
missingok
[...]
to
/var/log/haproxy/*.log {
daily
rotate 10
missingok
[...]
Now we can check if HAProxy logging is working.
tail -f /var/log/haproxy/haproxy.log
===================================================
global to have these messages end up in /var/log/haproxy.log you will need to:
1) configure syslog to accept network log events. This is done by adding the '-r' option to the SYSLOGD_OPTIONS in /etc/sysconfig/syslog
2) configure local2 events to go to the /var/log/haproxy.log file. A line like the following can be added to /etc/sysconfig/syslog
local2.* /var/log/haproxy.log
In haproxy conf file add
log 127.0.0.1 local2 info
Mysql replication-Master-Master
MySQL Master-Master replication.
Master-1 my.cnf configuration:
mkdir /var/lib/mysql/log/
log-bin=/var/lib/mysql/log/mysql-bin
log_warnings
log_slow_queries = /var/lib/mysql/log/slow.log
long_query_time = 5
log_long_format
tmpdir = /tmp
server-id = 1
log_slave_updates
replicate-same-server-id = 0
auto_increment_increment = 10
auto_increment_offset = 1
relay-log = mysql-relay-bin
Master-2 my.cnf configuration:
mkdir /var/lib/mysql/log/
log-bin=/var/lib/mysql/log/mysql-bin
log_warnings
log_slow_queries = /var/lib/mysql/log/slow.log
long_query_time = 5
log_long_format
tmpdir = /tmp
server-id = 2
replicate-same-server-id = 0
auto_increment_increment = 2
auto_increment_offset = 2
relay-log = mysql-relay-bin
First setup Master1 as Master and Master2 as slave for Master-1:
Follow below steps:
On Master-1:
grant replication slave on *.* to 'root'@'192.168.216.135' identified by 'admin';
show master status;
It shows file name and position, Use these records on Master-2 to run it as slave for Master-1.
Step 3: Now log on to master-2 and run the below query:
CHANGE MASTER TO MASTER_HOST='192.168.216.130', MASTER_USER='root',MASTER_PASSWORD='admin', MASTER_LOG_FILE='mysql-bin.000001',MASTER_LOG_POS=106;
Step 4: start slave
Step 5: show slave status \G
On this status, the following 2 records should be as follows
Slave_IO_Running: Yes
Slave_SQL_Running: Yes
These 2 records indicates Replication status. If these parameters show “Yes” that means replication is running successfully.
Setup Master2 as Master and Master1 as slave for Master-2:
On Master-2 server:
grant replication slave on *.* to 'root'@'192.168.216.130' identified by 'admin
Step 2: mysql> show master status;
Step 3: Now log on to master-1 and run the below query:
CHANGE MASTER TO MASTER_HOST='192.168.216.135', MASTER_USER='root',MASTER_PASSWORD='admin', MASTER_LOG_FILE='mysql-bin.000001',MASTER_LOG_POS=346;
Step 4: start slave
Step 5: show slave status \G
The following parameters should show “Yes”, so that replication is running successfully
Slave_IO_Running: Yes
Slave_SQL_Running: Yes
On both servers “slave_IO_Running” and “slave_SQL_Running” parameters should always be “Yes” for successful Master-Master Replication.
Master-1 my.cnf configuration:
mkdir /var/lib/mysql/log/
log-bin=/var/lib/mysql/log/mysql-bin
log_warnings
log_slow_queries = /var/lib/mysql/log/slow.log
long_query_time = 5
log_long_format
tmpdir = /tmp
server-id = 1
log_slave_updates
replicate-same-server-id = 0
auto_increment_increment = 10
auto_increment_offset = 1
relay-log = mysql-relay-bin
Master-2 my.cnf configuration:
mkdir /var/lib/mysql/log/
log-bin=/var/lib/mysql/log/mysql-bin
log_warnings
log_slow_queries = /var/lib/mysql/log/slow.log
long_query_time = 5
log_long_format
tmpdir = /tmp
server-id = 2
replicate-same-server-id = 0
auto_increment_increment = 2
auto_increment_offset = 2
relay-log = mysql-relay-bin
First setup Master1 as Master and Master2 as slave for Master-1:
Follow below steps:
On Master-1:
grant replication slave on *.* to 'root'@'192.168.216.135' identified by 'admin';
show master status;
It shows file name and position, Use these records on Master-2 to run it as slave for Master-1.
Step 3: Now log on to master-2 and run the below query:
CHANGE MASTER TO MASTER_HOST='192.168.216.130', MASTER_USER='root',MASTER_PASSWORD='admin', MASTER_LOG_FILE='mysql-bin.000001',MASTER_LOG_POS=106;
Step 4: start slave
Step 5: show slave status \G
On this status, the following 2 records should be as follows
Slave_IO_Running: Yes
Slave_SQL_Running: Yes
These 2 records indicates Replication status. If these parameters show “Yes” that means replication is running successfully.
Setup Master2 as Master and Master1 as slave for Master-2:
On Master-2 server:
grant replication slave on *.* to 'root'@'192.168.216.130' identified by 'admin
Step 2: mysql> show master status;
Step 3: Now log on to master-1 and run the below query:
CHANGE MASTER TO MASTER_HOST='192.168.216.135', MASTER_USER='root',MASTER_PASSWORD='admin', MASTER_LOG_FILE='mysql-bin.000001',MASTER_LOG_POS=346;
Step 4: start slave
Step 5: show slave status \G
The following parameters should show “Yes”, so that replication is running successfully
Slave_IO_Running: Yes
Slave_SQL_Running: Yes
On both servers “slave_IO_Running” and “slave_SQL_Running” parameters should always be “Yes” for successful Master-Master Replication.
Tuesday, June 24, 2014
Openstack Live Migration failure: operation failed
Error in the error log nova/compute
2014-06-25 01:32:50.752 2703 ERROR nova.virt.libvirt.driver [-]
[instance: fc118bff-77a3-4300-ab27-371a314b819f] Live Migration failure: operation failed: Failed to connect to remote libvirt URI qemu+tcp://compute2/system
Try updating the libvirt configurations. Modify /etc/libvirt/libvirtd.conf. To see all of the available options
2014-06-25 01:32:50.752 2703 ERROR nova.virt.libvirt.driver [-]
[instance: fc118bff-77a3-4300-ab27-371a314b819f] Live Migration failure: operation failed: Failed to connect to remote libvirt URI qemu+tcp://compute2/system
Try updating the libvirt configurations. Modify /etc/libvirt/libvirtd.conf. To see all of the available options
before : #listen_tls = 0
after : listen_tls = 0
before : #listen_tcp = 1
after : listen_tcp = 1
add: auth_tcp = "none"
Openstack+ Shared Storage(NFS) +Permission denied
While setting up Openstack with shared storage , If we get following error while creating a instance ensure that the server is in permissive selinux mode with
getenfonce
and if that too doesn't work try giving 755 permission to the mounted directory here its /var/lib/nova
2014-06-24 18:58:20.642 5119 TRACE nova.compute.manager [instance: a7996f1f-9af2-4410-8351-139d43f00786] libvirtError: internal error Process exited while reading console log output: qemu-kvm: -chardev file,id=charserial0,path=/var/lib/nova/instances/a7996f1f-9af2-4410-8351-139d43f00786/console.log: Could not open '/var/lib/nova/instances/a7996f1f-9af2-4410-8351-139d43f00786/console.log': Permission denied
At last if nothing else worked , tell libvirtd/qemu to use root user to access datas.
[root@compute nova]# cat /etc/libvirt/qemu.conf |grep root
user = "root"
#group = "root"
[root@compute nova]#
getenfonce
and if that too doesn't work try giving 755 permission to the mounted directory here its /var/lib/nova
2014-06-24 18:58:20.642 5119 TRACE nova.compute.manager [instance: a7996f1f-9af2-4410-8351-139d43f00786] libvirtError: internal error Process exited while reading console log output: qemu-kvm: -chardev file,id=charserial0,path=/var/lib/nova/instances/a7996f1f-9af2-4410-8351-139d43f00786/console.log: Could not open '/var/lib/nova/instances/a7996f1f-9af2-4410-8351-139d43f00786/console.log': Permission denied
At last if nothing else worked , tell libvirtd/qemu to use root user to access datas.
[root@compute nova]# cat /etc/libvirt/qemu.conf |grep root
user = "root"
#group = "root"
[root@compute nova]#
Subscribe to:
Posts (Atom)