基于Harbor构建企业级镜像仓库

1、环境准备

IP地址	主机名	所运行业务
192.168.252.14	harbor-01	harbor服务器
192.168.252.15	harbor-02	harbor服务器
192.168.252.16	share_storage	NFS、Redis、PostgreSQL

2、配置NFS服务器、redis、postgresql

1)安装NFS

[root@share_storage ~]# mkdir /data
[root@share_storage ~]# chmod o+x /data/
[root@share_storage ~]# yum install -y nfs-utils rpcbind

[root@share_storage ~]# cat /etc/exports
/harbordata	192.168.252.14(rw,no_root_squash)   192.168.252.15(rw,no_root_squash)

[root@share_storage ~]# systemctl enable --now nfs-server

[root@share_storage ~]# showmount -e localhost 
Export list for localhost:
/harbordata 192.168.252.15,192.168.252.14

2)配置harbor服务挂载NFS存储

[root@harbor-01 ~]# yum install -y nfs-utils

[root@harbor-01 ~]# tail -n1 /etc/fstab 
192.168.252.16:/harbordata	/data	nfs	defaults 0 0

[root@harbor-01 ~]# df -hT | grep nfs
192.168.252.16:/harbordata nfs4       18G  1.5G   16G    9% /data

3)部署redis服务

[root@share_storage ~]# wget -O /etc/yum.repos.d/epel.repo https://mirrors.aliyun.com/repo/epel-7.repo

[root@share_storage ~]# yum install -y redis

[root@share_storage ~]# vim /etc/redis.conf #修改监听地址
bind 192.168.140.12
daemonize yes

[root@share_storage ~]# systemctl enable --now redis
Created symlink from /etc/systemd/system/multi-user.target.wants/redis.service to /usr/lib/systemd/system/redis.service.
[root@share_storage ~]# netstat -tunlp | grep redis
tcp        0      0 192.168.252.16:6379     0.0.0.0:*               LISTEN      2646/redis-server 1

4)部署postgreSQL

安装postgreSQL

[root@share_storage ~]# yum install -y cmake gcc gcc-c++ perl readline readline-devel openssl openssl-devel zlib zlib-devel ncurses-devel readline readline-devel zlib zlib-devel
[root@share_storage ~]# wget --no-check-certificate https://ftp.postgresql.org/pub/source/v12.2/postgresql-12.2.tar.gz
[root@share_storage ~]# tar xf postgresql-12.2.tar.gz 
[root@share_storage ~]# cd postgresql-12.2/
[root@share_storage postgresql-12.2]# ./configure --prefix=/usr/local/postsql
[root@share_storage postgresql-12.2]# make && make install

初始化postgresql

[root@share_storage ~]# useradd postgres
[root@share_storage ~]# mkdir -p /work/harbor-db/{data,temp,log}
[root@share_storage ~]# chown -R postgres.postgres /work/harbor-db
[root@share_storage ~]# su postgres
[postgres@share_storage root]$ /usr/local/postsql/bin/initdb --username=postgres -D /work/harbor-db/data/

编辑postgreSQL配置文件(修改以下配置)

[root@share_storage ~]# vim /work/harbor-db/data/postgresql.conf
data_directory = '/work/harbor-db/data'

listen_addresses = '*'          
port = 5432                           
max_connections = 100   

unix_socket_directories = '/work/harbor-db/temp'
unix_socket_group = ''
unix_socket_permissions = 0777

shared_buffers = 128MB

timezone = 'Asia/Shanghai'
 
logging_collector = on
log_directory = '/work/harbor-db/log'
log_rotation_size = 1GB
log_timezone = 'Asia/Shanghai'
log_min_duration_statement = 100

指定允许远程连接数据库的客户端

[root@share_storage ~]# tail -n 2 /work/harbor-db/data/pg_hba.conf 
host	all		harbor		192.168.252.14/24	trust
host	all		harbor		192.168.252.15/24	trust

启动数据库

[postgres@share_storage ~]$ /usr/local/postsql/bin/pg_ctl -D /work/harbor-db/data/ -l /work/harbor-db/log/start.log start
[postgres@share_storage ~]$  netstat -antp | grep 5432
tcp        0      0 0.0.0.0:5432            0.0.0.0:*               LISTEN      25550/postgres

创建harbor需要的数据库及远程连接用户

[root@share_storage ~]# /usr/local/postsql/bin/psql -h 127.0.0.1 -p 5432 -U postgres
psql (12.2)
Type "help" for help.

postgres=# \password postgres
Enter new password: 
Enter it again: 
postgres=# create user harbor with password 'redhat';
CREATE ROLE
postgres=# CREATE DATABASE harbor;
CREATE DATABASE
postgres=# create database harbor_clair;
CREATE DATABASE
postgres=# create database harbor_notary_server;
CREATE DATABASE
postgres=# create database harbor_notary_signer; 
CREATE DATABASE
postgres=# GRANT ALL ON DATABASE harbor to harbor; 
GRANT
postgres=# GRANT ALL ON DATABASE harbor_clair to harbor; 
GRANT
postgres=# GRANT ALL ON DATABASE harbor_notary_server to harbor;
GRANT
postgres=# GRANT ALL ON DATABASE harbor_notary_signer to harbor;
GRANT

远程主机测试连接数据库（任意一台harbor即可）

[root@harbor_02 ~]# yum install -y postgresql
[root@harbor_02 ~]# psql -h 192.168.252.16 -p 5432 -U harbor -W

3、在两台harbor服务器上分别安装docker、docker-compose、harbor

安装docker Releases · goharbor/harbor (github.com)

cat << eof > /etc/yum.repos.d/docker-ce.repo
[docker-ce]
name=docker-ce
baseurl=https://repo.huaweicloud.com/docker-ce/linux/centos/7.9/x86_64/stable/
enabled=1
gpgcheck=0
eof

yum install -y docker-ce

sudo mkdir -p /etc/docker
sudo tee /etc/docker/daemon.json <<-'EOF'
{"registry-mirrors": ["https://s7kqknxt.mirror.aliyuncs.com"]
}
EOF
sudo systemctl daemon-reload
sudo systemctl restart docker

安装docker-compose Releases · docker/compose (github.com)

chmod a+x docker-compose
mv docker-compose /usr/local/bin/

安装harbor

[root@harbor-01 ~]# tar xf harbor-offline-installer-v2.2.2.tgz
[root@harbor-01 ~]# cp harbor/harbor.yml.tmpl harbor/harbor.yml
[root@harbor-01 ~]# vim harbor/harbor.yml
hostname: harbor.linux.com

注释https相关配置，本案例中没有配置证书
#https:
	# port: 443
	# certificate: /usr/local/harbor/ssl/harbor.ssl	
	# private_key: /usr/local/harbor/ssl/harbor.key

data_volume: /data

harbor_admin_password: Harbor12345


注释或删除harbor自带的数据库配置，后续配置外部数据库连接
#database:
	# password: root123
	# max_idle_conns: 50
	# max_open_conns: 1000

配置harbor连接外部postgresql数据库
external_database:
  harbor:
    host: 192.168.140.12
    port: 5432
    db_name: harbor
    username: harbor
    password: redhat
    ssl_mode: disable
    max_idle_conns: 2
    max_open_conns: 0
  notary_signer:
    host: 192.168.140.12
    port: 5432
    db_name: harbor_notary_signer
    username: harbor
    password: redhat
    ssl_mode: disable
  notary_server:
    host: 192.168.140.12
    port: 5432
    db_name: harbor_notary_server
    username: harbor
    password: redhat
    ssl_mode: disable

配置连接外部redis存储会话信息
external_redis:
   host: 192.168.140.12:6379
   password:
   
   registry_db_index: 1
   jobservice_db_index: 2
   chartmuseum_db_index: 3
   trivy_db_index: 5
   idle_timeout_seconds: 30

【可选】配置CA为harbor签发证书/生成V3证书

创建CA证书

mkdir /opt/ssl
cd /opt/ssl
openssl genrsa -out ca.key 4096
openssl req -x509 -new -nodes -sha512 -days 3650  -subj "/CN=harbor.linux.com"  -key ca.key  -out ca.crt

为harbor服务器创建证书

openssl genrsa -out server.key 4096
openssl req  -new -sha512  -subj "/CN=harbor.linux.com"  -key server.key  -out server.csr

创建v3.ext文件

[root@harbor-01 ssl]# cat v3.ext 
 authorityKeyIdentifier=keyid,issuer
 basicConstraints=CA:FALSE
 keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment
 extendedKeyUsage = serverAuth 
 subjectAltName = @alt_names
 [alt_names]
 DNS.1=harbor.linux.com

openssl x509 -req -sha512 -days 3650 -extfile v3.ext -CA ca.crt -CAkey ca.key -CAcreateserial -in server.csr -out server.crt

编辑harbor配置文件开启https访问就可以

启动harbor

[root@harbor-01 harbor]# ./prepare
[root@harbor-01 harbor]# ./install.sh

安装haproxy，keepalived实现高可用

两台harbor服务器的haproxy配置一样

[root@harbor-01 harbor]# yum install -y haproxy
[root@harbor-01 harbor]# sed '/^#/d' /etc/haproxy/haproxy.cfg 
global
    log         127.0.0.1 local2

    chroot      /var/lib/haproxy
    pidfile     /var/run/haproxy.pid
    maxconn     4000
    user        haproxy
    group       haproxy
    daemon

    # turn on stats unix socket
    stats socket /var/lib/haproxy/stats

defaults
    mode                    http
    log                     global
    option                  httplog
    option                  dontlognull
    option http-server-close
    option forwardfor       except 127.0.0.0/8
    option                  redispatch
    retries                 3
    timeout http-request    10s
    timeout queue           1m
    timeout connect         10s
    timeout client          1m
    timeout server          1m
    timeout http-keep-alive 10s
    timeout check           10s
    maxconn                 3000
frontend http-in
    bind *:8888
    default_backend servers

backend servers
    server server1 192.168.252.14:80 
    server server2 192.168.252.15:80

安装keepalived

主keepalived

[root@harbor-01 harbor]# yum install -y keepalived
[root@harbor-01 ~]# cat /etc/keepalived/keepalived.conf 
! Configuration File for keepalived

global_defs {
   router_id haproxy-01
}

# 定义外部脚本
vrrp_script check_harbor {
    script "/etc/keepalived/check_harbor.sh"	# 脚本名称
    interval 1					# 脚本执行周期 
}

vrrp_instance VI_1 {
    state MASTER
    interface ens33
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.252.100
    }
    # 调用外部脚本 
    track_script {
        check_harbor
    }
}
[root@harbor-01 ~]# cat /etc/keepalived/check_harbor.sh 
#!/bin/bash
#

netstat -tunlp | grep 80 &> /dev/null

if [ $? -ne 0 ]; then
    systemctl stop keepalived
fi

备keepalived

[root@harbor-02 ~]# cat /etc/keepalived/keepalived.conf 
! Configuration File for keepalived

global_defs {
   router_id haproxy-02
}

# 定义外部脚本
vrrp_script check_harbor {
    script "/etc/keepalived/check_harbor.sh"	# 脚本名称
    interval 1					# 脚本执行周期 
}

vrrp_instance VI_1 {
    state BACKUP
    interface ens33
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.252.100
    }
    # 调用外部脚本 
    track_script {
        check_harbor
    }
}
[root@harbor-02 ~]# cat /etc/keepalived/check_harbor.sh 
#!/bin/bash
#

netstat -tunlp | grep 80 &> /dev/null

if [ $? -ne 0 ]; then
    systemctl stop keepalived
fi

VIP地址只会在主节点显示，32位的掩码，给本地hosts文件添加VIP地址解析到harbor.linux.com可正常访问，使用ip a查看即可

测试keepalived故障转移

[root@harbor-01 ~]# cd harbor/
[root@harbor-01 harbor]# docker-compose stop

查看备keepalived VIP地址

[root@harbor-02 ~]# ip a | grep “192.168.252.100”
inet 192.168.252.100/32 scope global ens33

刷新浏览器可正常通过VIP访问