diff --git a/.drone.do.yml b/.drone.do.yml index 74a10cf..33a1ea4 100644 --- a/.drone.do.yml +++ b/.drone.do.yml @@ -1,5 +1,5 @@ --- - +# this builds and deploys the system on static ip servers kind: pipeline type: docker name: default @@ -8,7 +8,47 @@ clone: # skip_verify: true steps: -steps: +- name: printenv + when: + branch: + - do + image: appleboy/drone-ssh + environment: + LOCAL_DOCKER_REGISTRY: + from_secret: local-docker-registry + SSH_HOST: + from_secret: ssh-host + SSH_USER: + from_secret: ssh-user + SSH_PORT: + from_secret: ssh-port + CERTBOT_EMAIL: + from_secret: certbot-email + GIT_DOMAIN: + from_secret: git-domain + settings: + envs: + - local_docker_registry + - ssh_host + - ssh_user + - ssh_port + - certbot_email + - git_domain + host: + from_secret: ssh-host + port: + from_secret: ssh-port + username: + from_secret: ssh-user + password: + from_secret: ssh-password + script: + - echo SSH_HOST=$SSH_HOST > env + - echo SSH_USER=$SSH_USER >> env + - echo SSH_PORT=$SSH_PORT >> env + - echo LOCAL_DOCKER_REGISTRY=$LOCAL_DOCKER_REGISTRY >> env + - echo CERTBOT_EMAIL=$CERTBOT_EMAIL >> env + - echo GIT_DOMAIN=$GIT_DOMAIN >> env - name: wait when: branch: @@ -49,6 +89,22 @@ steps: commands: - cd letsencrypt-nginx - sh build.sh do $${LOCAL_DOCKER_REGISTRY} +- name: build-postgres + when: + branch: + - do + image: docker:dind + volumes: + - name: dockersock + path: /var/run + environment: + LOCAL_DOCKER_REGISTRY: + from_secret: local-docker-registry + commands: + - cd guacamole-postgresql + - docker build . -t $${LOCAL_DOCKER_REGISTRY}guacamole-postgresql + - docker push $${LOCAL_DOCKER_REGISTRY}guacamole-postgresql + - name: clear when: branch: @@ -107,6 +163,12 @@ steps: from_secret: certbot-email GIT_DOMAIN: from_secret: git-domain + GUACAMOLE_POSTGRES_USER: + from_secret: guacamole-postgres-user + GUACAMOLE_POSTGRES_DB: + from_secret: guacamole-postgres-db + GUACAMOLE_POSTGRES_PASSWORD: + from_secret: guacamole-postgres-password settings: envs: - drone_rpc_secret @@ -116,6 +178,9 @@ steps: - local_docker_registry - certbot_email - git_domain + - guacamole_postgres_user + - guacamole_postgres_db + - guacamole_postgres_password host: from_secret: ssh-host username: @@ -135,9 +200,13 @@ steps: - export SSH_USER=$SSH_USER - export CERTBOT_EMAIL=$CERTBOT_EMAIL - export GIT_DOMAIN=$GIT_DOMAIN + - export GUACAMOLE_POSTGRES_USER=$GUACAMOLE_POSTGRES_USER + - export GUACAMOLE_POSTGRES_DB=$GUACAMOLE_POSTGRES_DB + - export GUACAMOLE_POSTGRES_PASSWORD=$GUACAMOLE_POSTGRES_PASSWORD - docker network prune -f - cd ~/stack-deploy - docker pull $${LOCAL_DOCKER_REGISTRY}letsencrypt-do + - docker pull $${LOCAL_DOCKER_REGISTRY}guacamole-postgresql - docker stack rm gitea - sleep 60 - docker stack deploy -c docker-compose-do.yml gitea diff --git a/.drone.home.yml b/.drone.home.yml index 7cc8cb4..e13b318 100644 --- a/.drone.home.yml +++ b/.drone.home.yml @@ -1,5 +1,5 @@ --- - +# builds and deploys to stack using ngrok tunnel kind: pipeline type: docker name: default @@ -12,7 +12,39 @@ clone: # skip_verify: true steps: -steps: +- name: printenv + when: + branch: + - test-deploy-windows + image: appleboy/drone-ssh + environment: + LOCAL_DOCKER_REGISTRY: + from_secret: local-docker-registry + SSH_HOST: + from_secret: ssh-host + SSH_USER: + from_secret: ssh-user + SSH_PORT: + from_secret: ssh-port + settings: + envs: + - local_docker_registry + - ssh_host + - ssh_user + - ssh_port + host: + from_secret: ssh-host + port: + from_secret: ssh-port + username: + from_secret: ssh-user + password: + from_secret: ssh-password + script: + - echo SSH_HOST=$SSH_HOST > env + - echo SSH_USER=$SSH_USER >> env + - echo SSH_PORT=$SSH_PORT >> env + - echo LOCAL_DOCKER_REGISTRY=$LOCAL_DOCKER_REGISTRY >> env - name: wait when: branch: diff --git a/README.md b/README.md index b4a3ee8..1c52927 100644 --- a/README.md +++ b/README.md @@ -1,247 +1,30 @@ -# gitea in a stack with drone and guacamole +# stack -Remote system support. +* [Gitea](https://gitea.io/en-us/) +* [Drone](https://drone.io/) +* [Guacamole](https://guacamole.apache.org/) +* [Rocket Chat](https://rocket.chat/) -Consisting of +In a [docker](https://www.docker.com/) stack. -* gitea repository - github like self hoisted git and web application -* drone ci system -* guacamole - rdp, vnc and ssh over the internet in the browser - -## do +## static ip [![Build Status](https://sigyl.com:5000/api/badges/giles/stack/status.svg)](https://sigyl.com:5000/giles/stack) -## home +* home - https://sigyl.com/ +* git - https://sigyl.com/git/ +* drone - https://sigyl.com:5000/ +* guacamole - https://sigyl.com/remote/ +* chat - https://sigyl.com/chat/ + +[drone.do.yml](drone.do.yml) - [docker-compose-do.yml](docker-compose-do.yml) + +## tunnelled with ngrok + +(very slow if home internet) [![Build Status](https://drone.sigyl.com/api/badges/giles/stack/status.svg)](https://drone.sigyl.com/giles/stack) -(nb when self deploying with drone it will will stick on started and have to be cancelled. This is because the deployment tears down the previous running drone). - -NB each time you deploy this using drone it will leave an orphaned network for example: - -drone-yoZbiXiqssFcSsG0dP5d - -eventually this will start to cause an error with message: Docker “ERROR: could not find an available, non-overlapping IPv4 address pool among the defaults to assign to the network” - - -## installing docker - -Start with a fresh install of Ubuntu server 19.04 connected to the internet - -### update packages - -``` -sudo apt-get update -sudo apt-get upgrade -``` - -### remove old versions of docker - -(if it's a fresh install of linux there shouldn't be any) - -``` -sudo apt-get remove docker docker-engine docker.io -``` - -### install docker - -``` -sudo apt install docker.io -``` - -### add current user to docker group - -logout and back in afterwards - -``` -sudo usermod -aG docker $USER -``` - -### start and enable docker - -``` -sudo systemctl start docker -sudo systemctl enable docker -``` - -### change ssh port to 2022 - -``` -sudo vi /etc/ssh/sshd_config - -``` - -change Port 2022 - -### allow root to ssh - -``` -sudo vi /etc/ssh/sshd_config - -``` - -## set the root password - -``` -sudo passwd root -``` - -change PermitRootLogin yes - -reboot - -start a stack running gitea to host repository. - -## stack - -### labels - -get nodes with - -``` -docker node ls -``` - -add label with - -``` -docker node update --label-add com.sigyl.git-stack=yes [node id] -``` - -### generate certificates - -where [registry-domain] is the domain on which the registry will be served - - sh ca.sh [registry-domain]:5000 - - - sh make-cert.sh [registry-domain] registry - -### ngrok auth - -obtain ngrok auth token and place in .secrets in form - - authtoken: [token] - -### initial deploy - -```shell -export LOCAL_DOCKER_REGISTRY=registry.local-domain:5000 -export CERTBOT_EMAIL=giles.bradshaw@sigyl.com -export GIT_DOMAIN=git.sigyl.com -export REMOTE_DOMAIN=remote.sigyl.com -export DRONE_DOMAIN=drone.sigyl.com -export BLOG_DOMAIN=blog.sigyl.com - -export DRONE_RPC_SECRET=$(openssl rand -base64 48) -docker stack deploy -c docker-compose.yml gitea -``` - -### create gitea drone app with client id and secret - -### add secrets to repository in drone - - -![add secret](./add-secret.png) - -* blog-domain -* certbot-email -* drone-domain -* drone-gitea-client-id -* drone-gitea-client-secret -* drone-rpc-secret -* git-domain -* local-docker-registry -* remote-domain -* ssh-password -* ssh-port -* ssh-root-password -* ssh-root-user -* ssh-user -* ssh-host - -### kill orphan docker:dind containers - -Wen the system is deployed by pushing to repository the docker:dind container will be orphaned and will run forever unless killed.. - - -## guacamole - -np no spaces in postgres password - -docker stack for guacamole - -adapted from https://digitalmccullough.com/posts/setting-up-apache-guacamole-with-docker-stack.html - -### initialising - -find id - -``` -docker ps - -``` - -execute initdb.sql - -``` -docker exec -it [id] psql -U postgres -d guacamole_db -f /initdb.sql -docker exec -it [id] psql -U postgres -d guacamole_db -f /init-user.sql -v password='somepassword' -``` - -initial admin is guacadmin:guacadmin - -create a new admin and delete guacadmin - -## docker-exec-runner on windows - -These instructions are not very good... - -https://exec-runner.docs.drone.io/installation/windows/ - -download and unpack on linux with - -``` -curl -L https://github.com/drone-runners/drone-runner-exec/releases/latest/download/drone_runner_exec_windows_amd64.tar.gz | tar zx -``` - - -rename drone-runner-exec to drone-runner-exec.exe - -make directory c:\Drone\drone-runner-exec on windows - -copy drone-runner-exec.exe to directory - -make config file with - -``` - -DRONE_RPC_PROTO=https -DRONE_RPC_HOST=drone.sigyl.com:443 -DRONE_RPC_SECRET=[rpc secret] -DRONE_LOG_FILE=C:\Drone\drone-runner-exec\log.txt -DRONE_RUNNER_LABELS=web:true -``` - - -install and start service with - -``` -drone-runner-exec service install -drone-runner-exec service start -``` - -## chat - -Once the chat-mongo container is up you need to get its id and do - -``` -docker exec -it [id] mongo --eval "printjson(rs.initiate({_id: 'rs0', members: [ { _id: 0, host: 'localhost:27017' } ]}))" -``` - -then scale up chat - -``` -docker service scale gitea_chat=1 -``` \ No newline at end of file +* git - https://git.sigyl.com/ +* drone - https://drone.sigyl.com/ +* guacamole - https://remote.sigyl.com/ diff --git a/docker-compose-do.yml b/docker-compose-do.yml index 91d025e..3ed9215 100644 --- a/docker-compose-do.yml +++ b/docker-compose-do.yml @@ -14,11 +14,13 @@ services: - PROXY_PASS=http://gitea:3000/ - BLOG_PROXY_PASS=http://ghost:2368 - CHAT_PROXY_PASS=http://chat:3000 + - REMOTE_PROXY_PASS=http://guacamole:8080/guacamole/ - DRONE_PROXY_PASS=http://drone-server:8080 - REGISTRY_PROXY_PASS=http://registry:5000 - LOCATION=/git/ - BLOG_LOCATION=/blog/ - CHAT_LOCATION=/chat/ + - REMOTE_LOCATION=/remote/ volumes: - letsencrypt-git:/etc/letsencrypt networks: @@ -161,6 +163,49 @@ services: - REGISTRY_HTTP_ADDR=0.0.0.0:5000 networks: - appnet + guacamole-postgresql: + deploy: + placement: + constraints: [node.labels.com.sigyl.git-stack-chat == yes] + replicas: 1 + restart_policy: + condition: any + image: ${LOCAL_DOCKER_REGISTRY}guacamole-postgresql:latest + environment: + POSTGRES_PASSWORD: ${GUACAMOLE_POSTGRES_PASSWORD} + POSTGRES_DB: ${GUACAMOLE_POSTGRES_DB} + volumes: + - guacamole-postgresql-data:/var/lib/postgresql/data + networks: + - appnet + guacd: + deploy: + placement: + constraints: [node.labels.com.sigyl.git-stack-chat == yes] + replicas: 1 + restart_policy: + condition: any + image: guacamole/guacd:latest + networks: + - appnet + guacamole: + deploy: + placement: + constraints: [node.labels.com.sigyl.git-stack-chat == yes] + replicas: 1 + restart_policy: + condition: any + image: guacamole/guacamole:latest + environment: + - POSTGRES_HOSTNAME=guacamole-postgresql + - POSTGRES_PORT=5432 + - POSTGRES_USER=${GUACAMOLE_POSTGRES_USER} + - POSTGRES_PASSWORD=${GUACAMOLE_POSTGRES_PASSWORD} + - POSTGRES_DATABASE=${GUACAMOLE_POSTGRES_DB} + - GUACD_HOSTNAME=guacd + networks: + - appnet + volumes: gitea-db: gitea-app: @@ -171,6 +216,7 @@ volumes: drone: drone-data: registry-data: + guacamole-postgresql-data: networks: appnet: driver: overlay diff --git a/docker-compose-home.yml b/docker-compose-home.yml index 46db257..69cab60 100644 --- a/docker-compose-home.yml +++ b/docker-compose-home.yml @@ -257,11 +257,6 @@ services: POSTGRES_DB: guacamole_db volumes: - guacamole-postgresql-data:/var/lib/postgresql/data - #secrets: - # - source: guacamole-postgresql-password - # target: password - - #- /home/giles/guacamole-stack/initdb.sql:/initdb.sql networks: - appnet diff --git a/install.md b/install.md new file mode 100644 index 0000000..a6b4fee --- /dev/null +++ b/install.md @@ -0,0 +1,242 @@ +# gitea in a stack with drone and guacamole + +Remote system support. + +Consisting of + +* gitea repository - github like self hosted git and web application +* drone ci system +* guacamole - rdp, vnc and ssh over the internet in the browser + + +## home + + +(nb when self deploying with drone it will will stick on started and have to be cancelled. This is because the deployment tears down the previous running drone). + +NB each time you deploy this using drone it will leave an orphaned network for example: + +eventually this will start to cause an error with message: Docker “ERROR: could not find an available, non-overlapping IPv4 address pool among the defaults to assign to the network” + +You need to keep clearing out orphaned networks. + +## installing docker + +Start with a fresh install of Ubuntu server 19.04 connected to the internet + +### update packages + +``` +sudo apt-get update +sudo apt-get upgrade +``` + +### remove old versions of docker + +(if it's a fresh install of linux there shouldn't be any) + +``` +sudo apt-get remove docker docker-engine docker.io +``` + +### install docker + +``` +sudo apt install docker.io +``` + +### add current user to docker group + +logout and back in afterwards + +``` +sudo usermod -aG docker $USER +``` + +### start and enable docker + +``` +sudo systemctl start docker +sudo systemctl enable docker +``` + +### change ssh port to 2022 + +``` +sudo vi /etc/ssh/sshd_config + +``` + +change Port 2022 + +### allow root to ssh + +``` +sudo vi /etc/ssh/sshd_config + +``` + +## set the root password + +``` +sudo passwd root +``` + +change PermitRootLogin yes + +reboot + +start a stack running gitea to host repository. + +## stack + +### labels + +get nodes with + +``` +docker node ls +``` + +add label with + +``` +docker node update --label-add com.sigyl.git-stack=yes [node id] +``` + +### generate certificates + +where [registry-domain] is the domain on which the registry will be served + + sh ca.sh [registry-domain]:5000 + + + sh make-cert.sh [registry-domain] registry + +### ngrok auth + +obtain ngrok auth token and place in .secrets in form + + authtoken: [token] + +### initial deploy + +```shell +export LOCAL_DOCKER_REGISTRY=registry.local-domain:5000 +export CERTBOT_EMAIL=giles.bradshaw@sigyl.com +export GIT_DOMAIN=git.sigyl.com +export REMOTE_DOMAIN=remote.sigyl.com +export DRONE_DOMAIN=drone.sigyl.com +export BLOG_DOMAIN=blog.sigyl.com + +export DRONE_RPC_SECRET=$(openssl rand -base64 48) +docker stack deploy -c docker-compose.yml gitea +``` + +### create gitea drone app with client id and secret + +### add secrets to repository in drone + + +![add secret](./add-secret.png) + +* blog-domain +* certbot-email +* drone-domain +* drone-gitea-client-id +* drone-gitea-client-secret +* drone-rpc-secret +* git-domain +* local-docker-registry +* remote-domain +* ssh-password +* ssh-port +* ssh-root-password +* ssh-root-user +* ssh-user +* ssh-host + +### kill orphan docker:dind containers + +Wen the system is deployed by pushing to repository the docker:dind container will be orphaned and will run forever unless killed.. + + +## guacamole + +np no spaces in postgres password + +docker stack for guacamole + +adapted from https://digitalmccullough.com/posts/setting-up-apache-guacamole-with-docker-stack.html + +### initialising + +find id + +``` +docker ps + +``` + +execute initdb.sql + +``` +docker exec -it [id] psql -U postgres -d guacamole_db -f /initdb.sql +docker exec -it [id] psql -U postgres -d guacamole_db -f /init-user.sql -v password='somepassword' +``` + +initial admin is guacadmin:guacadmin + +create a new admin and delete guacadmin + +## docker-exec-runner on windows + +These instructions are not very good... + +https://exec-runner.docs.drone.io/installation/windows/ + +download and unpack on linux with + +``` +curl -L https://github.com/drone-runners/drone-runner-exec/releases/latest/download/drone_runner_exec_windows_amd64.tar.gz | tar zx +``` + + +rename drone-runner-exec to drone-runner-exec.exe + +make directory c:\Drone\drone-runner-exec on windows + +copy drone-runner-exec.exe to directory + +make config file with + +``` + +DRONE_RPC_PROTO=https +DRONE_RPC_HOST=drone.sigyl.com:443 +DRONE_RPC_SECRET=[rpc secret] +DRONE_LOG_FILE=C:\Drone\drone-runner-exec\log.txt +DRONE_RUNNER_LABELS=web:true +``` + + +install and start service with + +``` +drone-runner-exec service install +drone-runner-exec service start +``` + +## chat + +Once the chat-mongo container is up you need to get its id and do + +``` +docker exec -it [id] mongo --eval "printjson(rs.initiate({_id: 'rs0', members: [ { _id: 0, host: 'localhost:27017' } ]}))" +``` + +then scale up chat + +``` +docker service scale gitea_chat=1 +``` \ No newline at end of file diff --git a/letsencrypt-nginx/conf/do.conf b/letsencrypt-nginx/conf/do.conf index 95829ec..67f2341 100644 --- a/letsencrypt-nginx/conf/do.conf +++ b/letsencrypt-nginx/conf/do.conf @@ -163,15 +163,44 @@ location ${CHAT_LOCATION} { proxy_pass ${CHAT_PROXY_PASS}; proxy_http_version 1.1; - proxy_set_header Upgrade ${DOLLAR}http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_set_header Host ${DOLLAR}http_host; - proxy_set_header X-Real-IP ${DOLLAR}remote_addr; - proxy_set_header X-Forwarded-For ${DOLLAR}proxy_add_x_forwarded_for; - proxy_set_header X-Forward-Proto http; - proxy_set_header X-Nginx-Proxy true; - proxy_redirect off; + proxy_set_header Upgrade ${DOLLAR}http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host ${DOLLAR}http_host; + proxy_set_header X-Real-IP ${DOLLAR}remote_addr; + proxy_set_header X-Forwarded-For ${DOLLAR}proxy_add_x_forwarded_for; + proxy_set_header X-Forward-Proto http; + proxy_set_header X-Nginx-Proxy true; + proxy_redirect off; + } + location ${REMOTE_LOCATION}websocket-tunnel { + proxy_pass ${REMOTE_PROXY_PASS}websocket-tunnel; + proxy_http_version 1.1; + proxy_set_header Upgrade ${DOLLAR}http_upgrade; + proxy_set_header Connection "Upgrade"; + proxy_set_header Host ${DOLLAR}host; + proxy_set_header X-Real-IP ${DOLLAR}remote_addr; + proxy_set_header X-Forwarded-For ${DOLLAR}proxy_add_x_forwarded_for; + proxy_set_header X-Forward-Proto http; + proxy_set_header X-Nginx-Proxy true; + proxy_redirect off; + } + location ${REMOTE_LOCATION}websocket-tunnel/ { + proxy_pass ${REMOTE_PROXY_PASS}websocket-tunnel/; + proxy_http_version 1.1; + proxy_set_header Upgrade ${DOLLAR}http_upgrade; + proxy_set_header Connection "Upgrade"; + proxy_set_header Host ${DOLLAR}host; + proxy_set_header X-Real-IP ${DOLLAR}remote_addr; + proxy_set_header X-Forwarded-For ${DOLLAR}proxy_add_x_forwarded_for; + proxy_set_header X-Forward-Proto http; + proxy_set_header X-Nginx-Proxy true; + proxy_redirect off; + + } + + location ${REMOTE_LOCATION} { + proxy_pass ${REMOTE_PROXY_PASS}; } } diff --git a/letsencrypt-nginx/website/index.html b/letsencrypt-nginx/website/index.html index 22f99e9..f985d6b 100644 --- a/letsencrypt-nginx/website/index.html +++ b/letsencrypt-nginx/website/index.html @@ -1,22 +1,27 @@

SiGyl Ltd

+Bespoke software development. +

home