Merge branch 'do'

This commit is contained in:
Giles Bradshaw 2020-04-20 09:00:45 +01:00
commit 84c2cbdf98
8 changed files with 461 additions and 260 deletions

View File

@ -1,5 +1,5 @@
--- ---
# this builds and deploys the system on static ip servers
kind: pipeline kind: pipeline
type: docker type: docker
name: default name: default
@ -8,7 +8,47 @@ clone:
# skip_verify: true # skip_verify: true
steps: steps:
steps: - name: printenv
when:
branch:
- do
image: appleboy/drone-ssh
environment:
LOCAL_DOCKER_REGISTRY:
from_secret: local-docker-registry
SSH_HOST:
from_secret: ssh-host
SSH_USER:
from_secret: ssh-user
SSH_PORT:
from_secret: ssh-port
CERTBOT_EMAIL:
from_secret: certbot-email
GIT_DOMAIN:
from_secret: git-domain
settings:
envs:
- local_docker_registry
- ssh_host
- ssh_user
- ssh_port
- certbot_email
- git_domain
host:
from_secret: ssh-host
port:
from_secret: ssh-port
username:
from_secret: ssh-user
password:
from_secret: ssh-password
script:
- echo SSH_HOST=$SSH_HOST > env
- echo SSH_USER=$SSH_USER >> env
- echo SSH_PORT=$SSH_PORT >> env
- echo LOCAL_DOCKER_REGISTRY=$LOCAL_DOCKER_REGISTRY >> env
- echo CERTBOT_EMAIL=$CERTBOT_EMAIL >> env
- echo GIT_DOMAIN=$GIT_DOMAIN >> env
- name: wait - name: wait
when: when:
branch: branch:
@ -49,6 +89,22 @@ steps:
commands: commands:
- cd letsencrypt-nginx - cd letsencrypt-nginx
- sh build.sh do $${LOCAL_DOCKER_REGISTRY} - sh build.sh do $${LOCAL_DOCKER_REGISTRY}
- name: build-postgres
when:
branch:
- do
image: docker:dind
volumes:
- name: dockersock
path: /var/run
environment:
LOCAL_DOCKER_REGISTRY:
from_secret: local-docker-registry
commands:
- cd guacamole-postgresql
- docker build . -t $${LOCAL_DOCKER_REGISTRY}guacamole-postgresql
- docker push $${LOCAL_DOCKER_REGISTRY}guacamole-postgresql
- name: clear - name: clear
when: when:
branch: branch:
@ -107,6 +163,12 @@ steps:
from_secret: certbot-email from_secret: certbot-email
GIT_DOMAIN: GIT_DOMAIN:
from_secret: git-domain from_secret: git-domain
GUACAMOLE_POSTGRES_USER:
from_secret: guacamole-postgres-user
GUACAMOLE_POSTGRES_DB:
from_secret: guacamole-postgres-db
GUACAMOLE_POSTGRES_PASSWORD:
from_secret: guacamole-postgres-password
settings: settings:
envs: envs:
- drone_rpc_secret - drone_rpc_secret
@ -116,6 +178,9 @@ steps:
- local_docker_registry - local_docker_registry
- certbot_email - certbot_email
- git_domain - git_domain
- guacamole_postgres_user
- guacamole_postgres_db
- guacamole_postgres_password
host: host:
from_secret: ssh-host from_secret: ssh-host
username: username:
@ -135,9 +200,13 @@ steps:
- export SSH_USER=$SSH_USER - export SSH_USER=$SSH_USER
- export CERTBOT_EMAIL=$CERTBOT_EMAIL - export CERTBOT_EMAIL=$CERTBOT_EMAIL
- export GIT_DOMAIN=$GIT_DOMAIN - export GIT_DOMAIN=$GIT_DOMAIN
- export GUACAMOLE_POSTGRES_USER=$GUACAMOLE_POSTGRES_USER
- export GUACAMOLE_POSTGRES_DB=$GUACAMOLE_POSTGRES_DB
- export GUACAMOLE_POSTGRES_PASSWORD=$GUACAMOLE_POSTGRES_PASSWORD
- docker network prune -f - docker network prune -f
- cd ~/stack-deploy - cd ~/stack-deploy
- docker pull $${LOCAL_DOCKER_REGISTRY}letsencrypt-do - docker pull $${LOCAL_DOCKER_REGISTRY}letsencrypt-do
- docker pull $${LOCAL_DOCKER_REGISTRY}guacamole-postgresql
- docker stack rm gitea - docker stack rm gitea
- sleep 60 - sleep 60
- docker stack deploy -c docker-compose-do.yml gitea - docker stack deploy -c docker-compose-do.yml gitea

View File

@ -1,5 +1,5 @@
--- ---
# builds and deploys to stack using ngrok tunnel
kind: pipeline kind: pipeline
type: docker type: docker
name: default name: default
@ -12,7 +12,39 @@ clone:
# skip_verify: true # skip_verify: true
steps: steps:
steps: - name: printenv
when:
branch:
- test-deploy-windows
image: appleboy/drone-ssh
environment:
LOCAL_DOCKER_REGISTRY:
from_secret: local-docker-registry
SSH_HOST:
from_secret: ssh-host
SSH_USER:
from_secret: ssh-user
SSH_PORT:
from_secret: ssh-port
settings:
envs:
- local_docker_registry
- ssh_host
- ssh_user
- ssh_port
host:
from_secret: ssh-host
port:
from_secret: ssh-port
username:
from_secret: ssh-user
password:
from_secret: ssh-password
script:
- echo SSH_HOST=$SSH_HOST > env
- echo SSH_USER=$SSH_USER >> env
- echo SSH_PORT=$SSH_PORT >> env
- echo LOCAL_DOCKER_REGISTRY=$LOCAL_DOCKER_REGISTRY >> env
- name: wait - name: wait
when: when:
branch: branch:

259
README.md
View File

@ -1,247 +1,30 @@
# gitea in a stack with drone and guacamole # stack
Remote system support. * [Gitea](https://gitea.io/en-us/)
* [Drone](https://drone.io/)
* [Guacamole](https://guacamole.apache.org/)
* [Rocket Chat](https://rocket.chat/)
Consisting of In a [docker](https://www.docker.com/) stack.
* gitea repository - github like self hoisted git and web application ## static ip
* drone ci system
* guacamole - rdp, vnc and ssh over the internet in the browser
## do
[![Build Status](https://sigyl.com:5000/api/badges/giles/stack/status.svg)](https://sigyl.com:5000/giles/stack) [![Build Status](https://sigyl.com:5000/api/badges/giles/stack/status.svg)](https://sigyl.com:5000/giles/stack)
## home * home - https://sigyl.com/
* git - https://sigyl.com/git/
* drone - https://sigyl.com:5000/
* guacamole - https://sigyl.com/remote/
* chat - https://sigyl.com/chat/
[drone.do.yml](drone.do.yml) - [docker-compose-do.yml](docker-compose-do.yml)
## tunnelled with ngrok
(very slow if home internet)
[![Build Status](https://drone.sigyl.com/api/badges/giles/stack/status.svg)](https://drone.sigyl.com/giles/stack) [![Build Status](https://drone.sigyl.com/api/badges/giles/stack/status.svg)](https://drone.sigyl.com/giles/stack)
(nb when self deploying with drone it will will stick on started and have to be cancelled. This is because the deployment tears down the previous running drone). * git - https://git.sigyl.com/
* drone - https://drone.sigyl.com/
NB each time you deploy this using drone it will leave an orphaned network for example: * guacamole - https://remote.sigyl.com/
drone-yoZbiXiqssFcSsG0dP5d
eventually this will start to cause an error with message: Docker “ERROR: could not find an available, non-overlapping IPv4 address pool among the defaults to assign to the network”
## installing docker
Start with a fresh install of Ubuntu server 19.04 connected to the internet
### update packages
```
sudo apt-get update
sudo apt-get upgrade
```
### remove old versions of docker
(if it's a fresh install of linux there shouldn't be any)
```
sudo apt-get remove docker docker-engine docker.io
```
### install docker
```
sudo apt install docker.io
```
### add current user to docker group
logout and back in afterwards
```
sudo usermod -aG docker $USER
```
### start and enable docker
```
sudo systemctl start docker
sudo systemctl enable docker
```
### change ssh port to 2022
```
sudo vi /etc/ssh/sshd_config
```
change Port 2022
### allow root to ssh
```
sudo vi /etc/ssh/sshd_config
```
## set the root password
```
sudo passwd root
```
change PermitRootLogin yes
reboot
start a stack running gitea to host repository.
## stack
### labels
get nodes with
```
docker node ls
```
add label with
```
docker node update --label-add com.sigyl.git-stack=yes [node id]
```
### generate certificates
where [registry-domain] is the domain on which the registry will be served
sh ca.sh [registry-domain]:5000
sh make-cert.sh [registry-domain] registry
### ngrok auth
obtain ngrok auth token and place in .secrets in form
authtoken: [token]
### initial deploy
```shell
export LOCAL_DOCKER_REGISTRY=registry.local-domain:5000
export CERTBOT_EMAIL=giles.bradshaw@sigyl.com
export GIT_DOMAIN=git.sigyl.com
export REMOTE_DOMAIN=remote.sigyl.com
export DRONE_DOMAIN=drone.sigyl.com
export BLOG_DOMAIN=blog.sigyl.com
export DRONE_RPC_SECRET=$(openssl rand -base64 48)
docker stack deploy -c docker-compose.yml gitea
```
### create gitea drone app with client id and secret
### add secrets to repository in drone
![add secret](./add-secret.png)
* blog-domain
* certbot-email
* drone-domain
* drone-gitea-client-id
* drone-gitea-client-secret
* drone-rpc-secret
* git-domain
* local-docker-registry
* remote-domain
* ssh-password
* ssh-port
* ssh-root-password
* ssh-root-user
* ssh-user
* ssh-host
### kill orphan docker:dind containers
Wen the system is deployed by pushing to repository the docker:dind container will be orphaned and will run forever unless killed..
## guacamole
np no spaces in postgres password
docker stack for guacamole
adapted from https://digitalmccullough.com/posts/setting-up-apache-guacamole-with-docker-stack.html
### initialising
find id
```
docker ps
```
execute initdb.sql
```
docker exec -it [id] psql -U postgres -d guacamole_db -f /initdb.sql
docker exec -it [id] psql -U postgres -d guacamole_db -f /init-user.sql -v password='somepassword'
```
initial admin is guacadmin:guacadmin
create a new admin and delete guacadmin
## docker-exec-runner on windows
These instructions are not very good...
https://exec-runner.docs.drone.io/installation/windows/
download and unpack on linux with
```
curl -L https://github.com/drone-runners/drone-runner-exec/releases/latest/download/drone_runner_exec_windows_amd64.tar.gz | tar zx
```
rename drone-runner-exec to drone-runner-exec.exe
make directory c:\Drone\drone-runner-exec on windows
copy drone-runner-exec.exe to directory
make config file with
```
DRONE_RPC_PROTO=https
DRONE_RPC_HOST=drone.sigyl.com:443
DRONE_RPC_SECRET=[rpc secret]
DRONE_LOG_FILE=C:\Drone\drone-runner-exec\log.txt
DRONE_RUNNER_LABELS=web:true
```
install and start service with
```
drone-runner-exec service install
drone-runner-exec service start
```
## chat
Once the chat-mongo container is up you need to get its id and do
```
docker exec -it [id] mongo --eval "printjson(rs.initiate({_id: 'rs0', members: [ { _id: 0, host: 'localhost:27017' } ]}))"
```
then scale up chat
```
docker service scale gitea_chat=1
```

View File

@ -14,11 +14,13 @@ services:
- PROXY_PASS=http://gitea:3000/ - PROXY_PASS=http://gitea:3000/
- BLOG_PROXY_PASS=http://ghost:2368 - BLOG_PROXY_PASS=http://ghost:2368
- CHAT_PROXY_PASS=http://chat:3000 - CHAT_PROXY_PASS=http://chat:3000
- REMOTE_PROXY_PASS=http://guacamole:8080/guacamole/
- DRONE_PROXY_PASS=http://drone-server:8080 - DRONE_PROXY_PASS=http://drone-server:8080
- REGISTRY_PROXY_PASS=http://registry:5000 - REGISTRY_PROXY_PASS=http://registry:5000
- LOCATION=/git/ - LOCATION=/git/
- BLOG_LOCATION=/blog/ - BLOG_LOCATION=/blog/
- CHAT_LOCATION=/chat/ - CHAT_LOCATION=/chat/
- REMOTE_LOCATION=/remote/
volumes: volumes:
- letsencrypt-git:/etc/letsencrypt - letsencrypt-git:/etc/letsencrypt
networks: networks:
@ -161,6 +163,49 @@ services:
- REGISTRY_HTTP_ADDR=0.0.0.0:5000 - REGISTRY_HTTP_ADDR=0.0.0.0:5000
networks: networks:
- appnet - appnet
guacamole-postgresql:
deploy:
placement:
constraints: [node.labels.com.sigyl.git-stack-chat == yes]
replicas: 1
restart_policy:
condition: any
image: ${LOCAL_DOCKER_REGISTRY}guacamole-postgresql:latest
environment:
POSTGRES_PASSWORD: ${GUACAMOLE_POSTGRES_PASSWORD}
POSTGRES_DB: ${GUACAMOLE_POSTGRES_DB}
volumes:
- guacamole-postgresql-data:/var/lib/postgresql/data
networks:
- appnet
guacd:
deploy:
placement:
constraints: [node.labels.com.sigyl.git-stack-chat == yes]
replicas: 1
restart_policy:
condition: any
image: guacamole/guacd:latest
networks:
- appnet
guacamole:
deploy:
placement:
constraints: [node.labels.com.sigyl.git-stack-chat == yes]
replicas: 1
restart_policy:
condition: any
image: guacamole/guacamole:latest
environment:
- POSTGRES_HOSTNAME=guacamole-postgresql
- POSTGRES_PORT=5432
- POSTGRES_USER=${GUACAMOLE_POSTGRES_USER}
- POSTGRES_PASSWORD=${GUACAMOLE_POSTGRES_PASSWORD}
- POSTGRES_DATABASE=${GUACAMOLE_POSTGRES_DB}
- GUACD_HOSTNAME=guacd
networks:
- appnet
volumes: volumes:
gitea-db: gitea-db:
gitea-app: gitea-app:
@ -171,6 +216,7 @@ volumes:
drone: drone:
drone-data: drone-data:
registry-data: registry-data:
guacamole-postgresql-data:
networks: networks:
appnet: appnet:
driver: overlay driver: overlay

View File

@ -257,11 +257,6 @@ services:
POSTGRES_DB: guacamole_db POSTGRES_DB: guacamole_db
volumes: volumes:
- guacamole-postgresql-data:/var/lib/postgresql/data - guacamole-postgresql-data:/var/lib/postgresql/data
#secrets:
# - source: guacamole-postgresql-password
# target: password
#- /home/giles/guacamole-stack/initdb.sql:/initdb.sql
networks: networks:
- appnet - appnet

242
install.md Normal file
View File

@ -0,0 +1,242 @@
# gitea in a stack with drone and guacamole
Remote system support.
Consisting of
* gitea repository - github like self hosted git and web application
* drone ci system
* guacamole - rdp, vnc and ssh over the internet in the browser
## home
(nb when self deploying with drone it will will stick on started and have to be cancelled. This is because the deployment tears down the previous running drone).
NB each time you deploy this using drone it will leave an orphaned network for example:
eventually this will start to cause an error with message: Docker “ERROR: could not find an available, non-overlapping IPv4 address pool among the defaults to assign to the network”
You need to keep clearing out orphaned networks.
## installing docker
Start with a fresh install of Ubuntu server 19.04 connected to the internet
### update packages
```
sudo apt-get update
sudo apt-get upgrade
```
### remove old versions of docker
(if it's a fresh install of linux there shouldn't be any)
```
sudo apt-get remove docker docker-engine docker.io
```
### install docker
```
sudo apt install docker.io
```
### add current user to docker group
logout and back in afterwards
```
sudo usermod -aG docker $USER
```
### start and enable docker
```
sudo systemctl start docker
sudo systemctl enable docker
```
### change ssh port to 2022
```
sudo vi /etc/ssh/sshd_config
```
change Port 2022
### allow root to ssh
```
sudo vi /etc/ssh/sshd_config
```
## set the root password
```
sudo passwd root
```
change PermitRootLogin yes
reboot
start a stack running gitea to host repository.
## stack
### labels
get nodes with
```
docker node ls
```
add label with
```
docker node update --label-add com.sigyl.git-stack=yes [node id]
```
### generate certificates
where [registry-domain] is the domain on which the registry will be served
sh ca.sh [registry-domain]:5000
sh make-cert.sh [registry-domain] registry
### ngrok auth
obtain ngrok auth token and place in .secrets in form
authtoken: [token]
### initial deploy
```shell
export LOCAL_DOCKER_REGISTRY=registry.local-domain:5000
export CERTBOT_EMAIL=giles.bradshaw@sigyl.com
export GIT_DOMAIN=git.sigyl.com
export REMOTE_DOMAIN=remote.sigyl.com
export DRONE_DOMAIN=drone.sigyl.com
export BLOG_DOMAIN=blog.sigyl.com
export DRONE_RPC_SECRET=$(openssl rand -base64 48)
docker stack deploy -c docker-compose.yml gitea
```
### create gitea drone app with client id and secret
### add secrets to repository in drone
![add secret](./add-secret.png)
* blog-domain
* certbot-email
* drone-domain
* drone-gitea-client-id
* drone-gitea-client-secret
* drone-rpc-secret
* git-domain
* local-docker-registry
* remote-domain
* ssh-password
* ssh-port
* ssh-root-password
* ssh-root-user
* ssh-user
* ssh-host
### kill orphan docker:dind containers
Wen the system is deployed by pushing to repository the docker:dind container will be orphaned and will run forever unless killed..
## guacamole
np no spaces in postgres password
docker stack for guacamole
adapted from https://digitalmccullough.com/posts/setting-up-apache-guacamole-with-docker-stack.html
### initialising
find id
```
docker ps
```
execute initdb.sql
```
docker exec -it [id] psql -U postgres -d guacamole_db -f /initdb.sql
docker exec -it [id] psql -U postgres -d guacamole_db -f /init-user.sql -v password='somepassword'
```
initial admin is guacadmin:guacadmin
create a new admin and delete guacadmin
## docker-exec-runner on windows
These instructions are not very good...
https://exec-runner.docs.drone.io/installation/windows/
download and unpack on linux with
```
curl -L https://github.com/drone-runners/drone-runner-exec/releases/latest/download/drone_runner_exec_windows_amd64.tar.gz | tar zx
```
rename drone-runner-exec to drone-runner-exec.exe
make directory c:\Drone\drone-runner-exec on windows
copy drone-runner-exec.exe to directory
make config file with
```
DRONE_RPC_PROTO=https
DRONE_RPC_HOST=drone.sigyl.com:443
DRONE_RPC_SECRET=[rpc secret]
DRONE_LOG_FILE=C:\Drone\drone-runner-exec\log.txt
DRONE_RUNNER_LABELS=web:true
```
install and start service with
```
drone-runner-exec service install
drone-runner-exec service start
```
## chat
Once the chat-mongo container is up you need to get its id and do
```
docker exec -it [id] mongo --eval "printjson(rs.initiate({_id: 'rs0', members: [ { _id: 0, host: 'localhost:27017' } ]}))"
```
then scale up chat
```
docker service scale gitea_chat=1
```

View File

@ -163,15 +163,44 @@
location ${CHAT_LOCATION} { location ${CHAT_LOCATION} {
proxy_pass ${CHAT_PROXY_PASS}; proxy_pass ${CHAT_PROXY_PASS};
proxy_http_version 1.1; proxy_http_version 1.1;
proxy_set_header Upgrade ${DOLLAR}http_upgrade; proxy_set_header Upgrade ${DOLLAR}http_upgrade;
proxy_set_header Connection "upgrade"; proxy_set_header Connection "upgrade";
proxy_set_header Host ${DOLLAR}http_host; proxy_set_header Host ${DOLLAR}http_host;
proxy_set_header X-Real-IP ${DOLLAR}remote_addr; proxy_set_header X-Real-IP ${DOLLAR}remote_addr;
proxy_set_header X-Forwarded-For ${DOLLAR}proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-For ${DOLLAR}proxy_add_x_forwarded_for;
proxy_set_header X-Forward-Proto http; proxy_set_header X-Forward-Proto http;
proxy_set_header X-Nginx-Proxy true; proxy_set_header X-Nginx-Proxy true;
proxy_redirect off; proxy_redirect off;
}
location ${REMOTE_LOCATION}websocket-tunnel {
proxy_pass ${REMOTE_PROXY_PASS}websocket-tunnel;
proxy_http_version 1.1;
proxy_set_header Upgrade ${DOLLAR}http_upgrade;
proxy_set_header Connection "Upgrade";
proxy_set_header Host ${DOLLAR}host;
proxy_set_header X-Real-IP ${DOLLAR}remote_addr;
proxy_set_header X-Forwarded-For ${DOLLAR}proxy_add_x_forwarded_for;
proxy_set_header X-Forward-Proto http;
proxy_set_header X-Nginx-Proxy true;
proxy_redirect off;
}
location ${REMOTE_LOCATION}websocket-tunnel/ {
proxy_pass ${REMOTE_PROXY_PASS}websocket-tunnel/;
proxy_http_version 1.1;
proxy_set_header Upgrade ${DOLLAR}http_upgrade;
proxy_set_header Connection "Upgrade";
proxy_set_header Host ${DOLLAR}host;
proxy_set_header X-Real-IP ${DOLLAR}remote_addr;
proxy_set_header X-Forwarded-For ${DOLLAR}proxy_add_x_forwarded_for;
proxy_set_header X-Forward-Proto http;
proxy_set_header X-Nginx-Proxy true;
proxy_redirect off;
}
location ${REMOTE_LOCATION} {
proxy_pass ${REMOTE_PROXY_PASS};
} }
} }

View File

@ -1,22 +1,27 @@
<h1>SiGyl Ltd</h1> <h1>SiGyl Ltd</h1>
Bespoke software development.
<ul> <ul>
<li> <li>
<a href='https://sigyl.com/git'>repos</a> <a href='https://sigyl.com/git'>git</a>
</li>
<li>
<a href='https://sigyl.com/chat'>chat</a>
</li> </li>
<li> <li>
<a href='https://sigyl.com:5000'>drone</a> <a href='https://sigyl.com:5000'>drone</a>
</li> </li>
<li>
<a href='https://sigyl.com/remote'>remote</a>
</li>
<li>
<a href='https://sigyl.com/chat'>chat</a>
</li>
</ul> </ul>
<h2>home</h2> <h2>home</h2>
<ul> <ul>
<li> <li>
<a href='https://git.sigyl.com'>repos</a> <a href='https://git.sigyl.com'>git</a>
</li> </li>
<li> <li>
<a href='https://drone.sigyl.com'>drone</a> <a href='https://drone.sigyl.com'>drone</a>