Merge branch 'do'

.drone.do.yml

@@ -1,5 +1,5 @@
 ---
-
+# this builds and deploys the system on static ip servers
 kind: pipeline
 type: docker
 name: default
@@ -8,7 +8,47 @@ clone:
   # skip_verify: true
 
 steps:
-steps:
+- name: printenv
+  when:
+    branch:
+    - do
+  image: appleboy/drone-ssh
+  environment:
+    LOCAL_DOCKER_REGISTRY:
+      from_secret: local-docker-registry
+    SSH_HOST:
+      from_secret: ssh-host
+    SSH_USER:
+      from_secret: ssh-user
+    SSH_PORT:
+      from_secret: ssh-port
+    CERTBOT_EMAIL:
+      from_secret: certbot-email
+    GIT_DOMAIN:
+      from_secret: git-domain
+  settings:
+    envs:
+      - local_docker_registry
+      - ssh_host
+      - ssh_user
+      - ssh_port
+      - certbot_email
+      - git_domain
+    host:
+      from_secret: ssh-host
+    port:
+      from_secret: ssh-port
+    username: 
+      from_secret: ssh-user
+    password:
+      from_secret: ssh-password
+    script:
+      - echo SSH_HOST=$SSH_HOST > env
+      - echo SSH_USER=$SSH_USER >> env
+      - echo SSH_PORT=$SSH_PORT >> env
+      - echo LOCAL_DOCKER_REGISTRY=$LOCAL_DOCKER_REGISTRY >> env
+      - echo CERTBOT_EMAIL=$CERTBOT_EMAIL >> env
+      - echo GIT_DOMAIN=$GIT_DOMAIN >> env
 - name: wait
   when:
     branch:
@@ -49,6 +89,22 @@ steps:
   commands:
   - cd letsencrypt-nginx
   - sh build.sh do $${LOCAL_DOCKER_REGISTRY}
+- name: build-postgres
+  when:
+    branch:
+    - do
+  image: docker:dind
+  volumes:
+  - name: dockersock
+    path: /var/run
+  environment:
+    LOCAL_DOCKER_REGISTRY:
+      from_secret: local-docker-registry
+  commands:
+  - cd guacamole-postgresql
+  - docker build . -t $${LOCAL_DOCKER_REGISTRY}guacamole-postgresql
+  - docker push $${LOCAL_DOCKER_REGISTRY}guacamole-postgresql
+
 - name: clear
   when:
     branch:
@@ -107,6 +163,12 @@ steps:
       from_secret: certbot-email
     GIT_DOMAIN:
       from_secret: git-domain
+    GUACAMOLE_POSTGRES_USER:
+      from_secret: guacamole-postgres-user
+    GUACAMOLE_POSTGRES_DB:
+      from_secret: guacamole-postgres-db
+    GUACAMOLE_POSTGRES_PASSWORD:
+      from_secret: guacamole-postgres-password
   settings:
     envs:
       - drone_rpc_secret
@@ -116,6 +178,9 @@ steps:
       - local_docker_registry
       - certbot_email
       - git_domain
+      - guacamole_postgres_user
+      - guacamole_postgres_db
+      - guacamole_postgres_password
     host:
       from_secret: ssh-host
     username:
@@ -135,9 +200,13 @@ steps:
       - export SSH_USER=$SSH_USER
       - export CERTBOT_EMAIL=$CERTBOT_EMAIL
       - export GIT_DOMAIN=$GIT_DOMAIN
+      - export GUACAMOLE_POSTGRES_USER=$GUACAMOLE_POSTGRES_USER
+      - export GUACAMOLE_POSTGRES_DB=$GUACAMOLE_POSTGRES_DB
+      - export GUACAMOLE_POSTGRES_PASSWORD=$GUACAMOLE_POSTGRES_PASSWORD
       - docker network prune -f
       - cd ~/stack-deploy
       - docker pull $${LOCAL_DOCKER_REGISTRY}letsencrypt-do
+      - docker pull $${LOCAL_DOCKER_REGISTRY}guacamole-postgresql
       - docker stack rm gitea
       - sleep 60
       - docker stack deploy -c docker-compose-do.yml gitea

.drone.home.yml

@@ -1,5 +1,5 @@
 ---
-
+# builds and deploys to stack using ngrok tunnel
 kind: pipeline
 type: docker
 name: default
@@ -12,7 +12,39 @@ clone:
   # skip_verify: true
 
 steps:
-steps:
+- name: printenv
+  when:
+    branch:
+    - test-deploy-windows
+  image: appleboy/drone-ssh
+  environment:
+    LOCAL_DOCKER_REGISTRY:
+      from_secret: local-docker-registry
+    SSH_HOST:
+      from_secret: ssh-host
+    SSH_USER:
+      from_secret: ssh-user
+    SSH_PORT:
+      from_secret: ssh-port
+  settings:
+    envs:
+      - local_docker_registry
+      - ssh_host
+      - ssh_user
+      - ssh_port
+    host:
+      from_secret: ssh-host
+    port:
+      from_secret: ssh-port
+    username: 
+      from_secret: ssh-user
+    password:
+      from_secret: ssh-password
+    script:
+      - echo SSH_HOST=$SSH_HOST > env
+      - echo SSH_USER=$SSH_USER >> env
+      - echo SSH_PORT=$SSH_PORT >> env
+      - echo LOCAL_DOCKER_REGISTRY=$LOCAL_DOCKER_REGISTRY >> env
 - name: wait
   when:
     branch:

README.md

@@ -1,247 +1,30 @@
-# gitea in a stack with drone and guacamole
+# stack
 
-Remote system support.
+* [Gitea](https://gitea.io/en-us/)
+* [Drone](https://drone.io/)
+* [Guacamole](https://guacamole.apache.org/)
+* [Rocket Chat](https://rocket.chat/)
 
-Consisting of
+In a [docker](https://www.docker.com/) stack.
 
-* gitea repository - github like self hoisted git and web application
+## static ip
-* drone ci system
-* guacamole - rdp, vnc and ssh over the internet in the browser
-
-## do
 
 [![Build Status](https://sigyl.com:5000/api/badges/giles/stack/status.svg)](https://sigyl.com:5000/giles/stack)
 
-## home
+* home - https://sigyl.com/
+* git - https://sigyl.com/git/
+* drone - https://sigyl.com:5000/
+* guacamole - https://sigyl.com/remote/
+* chat - https://sigyl.com/chat/
+
+[drone.do.yml](drone.do.yml) - [docker-compose-do.yml](docker-compose-do.yml)
+
+## tunnelled with ngrok
+
+(very slow if home internet)
 
 [![Build Status](https://drone.sigyl.com/api/badges/giles/stack/status.svg)](https://drone.sigyl.com/giles/stack)
 
-(nb when self deploying with drone it will will stick on started and have to be cancelled.  This is because the deployment tears down the previous running drone).
+* git - https://git.sigyl.com/
-
+* drone - https://drone.sigyl.com/
-NB each time you deploy this using drone it will leave an orphaned network for example:
+* guacamole - https://remote.sigyl.com/
-
-drone-yoZbiXiqssFcSsG0dP5d
-
-eventually this will start to cause an error with message: Docker “ERROR: could not find an available, non-overlapping IPv4 address pool among the defaults to assign to the network”
-
-
-## installing docker
-
-Start with a fresh install of Ubuntu server 19.04 connected to the internet
-
-### update packages
-
-```
-sudo apt-get update
-sudo apt-get upgrade
-```
-
-### remove old versions of docker
-
-(if it's a fresh install of linux there shouldn't be any)
-
-```
-sudo apt-get remove docker docker-engine docker.io
-```
-
-### install docker
-
-```
-sudo apt install docker.io
-```
-
-### add current user to docker group
-
-logout and back in afterwards
-
-```
-sudo usermod -aG docker $USER
-```
-
-### start and enable docker
-
-```
-sudo systemctl start docker
-sudo systemctl enable docker
-```
-
-### change ssh port to 2022
-
-```
-sudo vi /etc/ssh/sshd_config
-
-```
-
-change Port 2022
-
-### allow root to ssh
-
-```
-sudo vi /etc/ssh/sshd_config
-
-```
-
-## set the root password
-
-```
-sudo passwd root
-```
-
-change PermitRootLogin yes
-
-reboot
-
-start a stack running gitea to host repository.
-
-## stack
-
-### labels
-
-get nodes with
-
-```
-docker node ls
-```
-
-add label with
-
-```
-docker node update --label-add com.sigyl.git-stack=yes [node id]
-```
-
-### generate certificates
-
-where [registry-domain] is the domain on which the registry will be served
-
-    sh ca.sh [registry-domain]:5000
-
-
-    sh make-cert.sh [registry-domain] registry
-
-### ngrok auth
-
-obtain ngrok auth token and place in .secrets in form
-
-    authtoken: [token]
-
-### initial deploy
-
-```shell
-export LOCAL_DOCKER_REGISTRY=registry.local-domain:5000
-export CERTBOT_EMAIL=giles.bradshaw@sigyl.com
-export GIT_DOMAIN=git.sigyl.com
-export REMOTE_DOMAIN=remote.sigyl.com
-export DRONE_DOMAIN=drone.sigyl.com
-export BLOG_DOMAIN=blog.sigyl.com
-
-export DRONE_RPC_SECRET=$(openssl rand -base64 48)
-docker stack deploy -c docker-compose.yml gitea
-```
-
-### create gitea drone app with client id and secret
-
-### add secrets to repository in drone
-
-
-![add secret](./add-secret.png)
-
-* blog-domain
-* certbot-email
-* drone-domain
-* drone-gitea-client-id
-* drone-gitea-client-secret
-* drone-rpc-secret
-* git-domain
-* local-docker-registry
-* remote-domain
-* ssh-password
-* ssh-port
-* ssh-root-password
-* ssh-root-user
-* ssh-user
-* ssh-host
-
-### kill orphan docker:dind containers
-
-Wen the system is deployed by pushing to repository the docker:dind container will be orphaned and will run forever unless killed..
-
-
-## guacamole
-
-np no spaces in postgres password
-
-docker stack for guacamole
-
-adapted from https://digitalmccullough.com/posts/setting-up-apache-guacamole-with-docker-stack.html
-
-### initialising
-
-find id
-
-```
-docker ps
-
-```
-
-execute initdb.sql
-
-```
-docker exec -it [id] psql -U postgres -d guacamole_db -f /initdb.sql
-docker exec -it [id] psql -U postgres -d guacamole_db -f /init-user.sql -v password='somepassword'
-```
-
-initial admin is guacadmin:guacadmin
-
-create a new admin and delete guacadmin
-
-## docker-exec-runner on windows
-
-These instructions are not very good...
-
-https://exec-runner.docs.drone.io/installation/windows/
-
-download and unpack on linux with
-
-```
-curl -L https://github.com/drone-runners/drone-runner-exec/releases/latest/download/drone_runner_exec_windows_amd64.tar.gz | tar zx
-```
-
-
-rename drone-runner-exec to drone-runner-exec.exe
-
-make directory c:\Drone\drone-runner-exec on windows
-
-copy drone-runner-exec.exe to directory
-
-make config file with
-
-```
-
-DRONE_RPC_PROTO=https
-DRONE_RPC_HOST=drone.sigyl.com:443
-DRONE_RPC_SECRET=[rpc secret]
-DRONE_LOG_FILE=C:\Drone\drone-runner-exec\log.txt
-DRONE_RUNNER_LABELS=web:true
-```
-
-
-install and start service with 
-
-```
-drone-runner-exec service install
-drone-runner-exec service start
-```
-
-## chat
-
-Once the chat-mongo container is up you need to get its id and do
-
-```
-docker exec -it [id] mongo --eval "printjson(rs.initiate({_id: 'rs0', members: [ { _id: 0, host: 'localhost:27017' } ]}))"
-```
-
-then scale up chat
-
-```
-docker service scale gitea_chat=1
-```

docker-compose-do.yml

@@ -14,11 +14,13 @@ services:
       - PROXY_PASS=http://gitea:3000/
       - BLOG_PROXY_PASS=http://ghost:2368
       - CHAT_PROXY_PASS=http://chat:3000
+      - REMOTE_PROXY_PASS=http://guacamole:8080/guacamole/
       - DRONE_PROXY_PASS=http://drone-server:8080
       - REGISTRY_PROXY_PASS=http://registry:5000
       - LOCATION=/git/
       - BLOG_LOCATION=/blog/
       - CHAT_LOCATION=/chat/
+      - REMOTE_LOCATION=/remote/
     volumes:
       - letsencrypt-git:/etc/letsencrypt
     networks:
@@ -161,6 +163,49 @@ services:
       - REGISTRY_HTTP_ADDR=0.0.0.0:5000
     networks:
       - appnet
+  guacamole-postgresql:
+    deploy:
+      placement:
+        constraints: [node.labels.com.sigyl.git-stack-chat == yes]
+      replicas: 1
+      restart_policy:
+        condition: any
+    image: ${LOCAL_DOCKER_REGISTRY}guacamole-postgresql:latest
+    environment:
+      POSTGRES_PASSWORD: ${GUACAMOLE_POSTGRES_PASSWORD}
+      POSTGRES_DB: ${GUACAMOLE_POSTGRES_DB}
+    volumes:
+      - guacamole-postgresql-data:/var/lib/postgresql/data
+    networks:
+      - appnet
+  guacd:
+    deploy:
+      placement:
+        constraints: [node.labels.com.sigyl.git-stack-chat == yes]
+      replicas: 1
+      restart_policy:
+        condition: any
+    image: guacamole/guacd:latest
+    networks:
+      - appnet
+  guacamole:
+    deploy:
+      placement:
+        constraints: [node.labels.com.sigyl.git-stack-chat == yes]
+      replicas: 1
+      restart_policy:
+        condition: any
+    image: guacamole/guacamole:latest
+    environment:
+      - POSTGRES_HOSTNAME=guacamole-postgresql
+      - POSTGRES_PORT=5432
+      - POSTGRES_USER=${GUACAMOLE_POSTGRES_USER}
+      - POSTGRES_PASSWORD=${GUACAMOLE_POSTGRES_PASSWORD}
+      - POSTGRES_DATABASE=${GUACAMOLE_POSTGRES_DB}
+      - GUACD_HOSTNAME=guacd
+    networks:
+      - appnet
+
 volumes:
   gitea-db:
   gitea-app:
@@ -171,6 +216,7 @@ volumes:
   drone:
   drone-data:
   registry-data:
+  guacamole-postgresql-data:
 networks:
   appnet:
     driver: overlay

docker-compose-home.yml

@@ -257,11 +257,6 @@ services:
       POSTGRES_DB: guacamole_db
     volumes:
       - guacamole-postgresql-data:/var/lib/postgresql/data
-    #secrets:
-    #  - source: guacamole-postgresql-password
-    #    target: password
-
-      #- /home/giles/guacamole-stack/initdb.sql:/initdb.sql
     networks:
       - appnet
 

install.md

@@ -0,0 +1,242 @@
+# gitea in a stack with drone and guacamole
+
+Remote system support.
+
+Consisting of
+
+* gitea repository - github like self hosted git and web application
+* drone ci system
+* guacamole - rdp, vnc and ssh over the internet in the browser
+
+
+## home
+
+
+(nb when self deploying with drone it will will stick on started and have to be cancelled.  This is because the deployment tears down the previous running drone).
+
+NB each time you deploy this using drone it will leave an orphaned network for example:
+
+eventually this will start to cause an error with message: Docker “ERROR: could not find an available, non-overlapping IPv4 address pool among the defaults to assign to the network”
+
+You need to keep clearing out orphaned networks.
+
+## installing docker
+
+Start with a fresh install of Ubuntu server 19.04 connected to the internet
+
+### update packages
+
+```
+sudo apt-get update
+sudo apt-get upgrade
+```
+
+### remove old versions of docker
+
+(if it's a fresh install of linux there shouldn't be any)
+
+```
+sudo apt-get remove docker docker-engine docker.io
+```
+
+### install docker
+
+```
+sudo apt install docker.io
+```
+
+### add current user to docker group
+
+logout and back in afterwards
+
+```
+sudo usermod -aG docker $USER
+```
+
+### start and enable docker
+
+```
+sudo systemctl start docker
+sudo systemctl enable docker
+```
+
+### change ssh port to 2022
+
+```
+sudo vi /etc/ssh/sshd_config
+
+```
+
+change Port 2022
+
+### allow root to ssh
+
+```
+sudo vi /etc/ssh/sshd_config
+
+```
+
+## set the root password
+
+```
+sudo passwd root
+```
+
+change PermitRootLogin yes
+
+reboot
+
+start a stack running gitea to host repository.
+
+## stack
+
+### labels
+
+get nodes with
+
+```
+docker node ls
+```
+
+add label with
+
+```
+docker node update --label-add com.sigyl.git-stack=yes [node id]
+```
+
+### generate certificates
+
+where [registry-domain] is the domain on which the registry will be served
+
+    sh ca.sh [registry-domain]:5000
+
+
+    sh make-cert.sh [registry-domain] registry
+
+### ngrok auth
+
+obtain ngrok auth token and place in .secrets in form
+
+    authtoken: [token]
+
+### initial deploy
+
+```shell
+export LOCAL_DOCKER_REGISTRY=registry.local-domain:5000
+export CERTBOT_EMAIL=giles.bradshaw@sigyl.com
+export GIT_DOMAIN=git.sigyl.com
+export REMOTE_DOMAIN=remote.sigyl.com
+export DRONE_DOMAIN=drone.sigyl.com
+export BLOG_DOMAIN=blog.sigyl.com
+
+export DRONE_RPC_SECRET=$(openssl rand -base64 48)
+docker stack deploy -c docker-compose.yml gitea
+```
+
+### create gitea drone app with client id and secret
+
+### add secrets to repository in drone
+
+
+![add secret](./add-secret.png)
+
+* blog-domain
+* certbot-email
+* drone-domain
+* drone-gitea-client-id
+* drone-gitea-client-secret
+* drone-rpc-secret
+* git-domain
+* local-docker-registry
+* remote-domain
+* ssh-password
+* ssh-port
+* ssh-root-password
+* ssh-root-user
+* ssh-user
+* ssh-host
+
+### kill orphan docker:dind containers
+
+Wen the system is deployed by pushing to repository the docker:dind container will be orphaned and will run forever unless killed..
+
+
+## guacamole
+
+np no spaces in postgres password
+
+docker stack for guacamole
+
+adapted from https://digitalmccullough.com/posts/setting-up-apache-guacamole-with-docker-stack.html
+
+### initialising
+
+find id
+
+```
+docker ps
+
+```
+
+execute initdb.sql
+
+```
+docker exec -it [id] psql -U postgres -d guacamole_db -f /initdb.sql
+docker exec -it [id] psql -U postgres -d guacamole_db -f /init-user.sql -v password='somepassword'
+```
+
+initial admin is guacadmin:guacadmin
+
+create a new admin and delete guacadmin
+
+## docker-exec-runner on windows
+
+These instructions are not very good...
+
+https://exec-runner.docs.drone.io/installation/windows/
+
+download and unpack on linux with
+
+```
+curl -L https://github.com/drone-runners/drone-runner-exec/releases/latest/download/drone_runner_exec_windows_amd64.tar.gz | tar zx
+```
+
+
+rename drone-runner-exec to drone-runner-exec.exe
+
+make directory c:\Drone\drone-runner-exec on windows
+
+copy drone-runner-exec.exe to directory
+
+make config file with
+
+```
+
+DRONE_RPC_PROTO=https
+DRONE_RPC_HOST=drone.sigyl.com:443
+DRONE_RPC_SECRET=[rpc secret]
+DRONE_LOG_FILE=C:\Drone\drone-runner-exec\log.txt
+DRONE_RUNNER_LABELS=web:true
+```
+
+
+install and start service with 
+
+```
+drone-runner-exec service install
+drone-runner-exec service start
+```
+
+## chat
+
+Once the chat-mongo container is up you need to get its id and do
+
+```
+docker exec -it [id] mongo --eval "printjson(rs.initiate({_id: 'rs0', members: [ { _id: 0, host: 'localhost:27017' } ]}))"
+```
+
+then scale up chat
+
+```
+docker service scale gitea_chat=1
+```

letsencrypt-nginx/conf/do.conf

@@ -163,15 +163,44 @@
         location ${CHAT_LOCATION} {
            proxy_pass ${CHAT_PROXY_PASS};
            proxy_http_version 1.1;
-        proxy_set_header Upgrade ${DOLLAR}http_upgrade;
+           proxy_set_header Upgrade ${DOLLAR}http_upgrade;
-        proxy_set_header Connection "upgrade";
+           proxy_set_header Connection "upgrade";
-        proxy_set_header Host ${DOLLAR}http_host;
+           proxy_set_header Host ${DOLLAR}http_host;
-        proxy_set_header X-Real-IP ${DOLLAR}remote_addr;
+           proxy_set_header X-Real-IP ${DOLLAR}remote_addr;
-        proxy_set_header X-Forwarded-For ${DOLLAR}proxy_add_x_forwarded_for;
+           proxy_set_header X-Forwarded-For ${DOLLAR}proxy_add_x_forwarded_for;
-        proxy_set_header X-Forward-Proto http;
+           proxy_set_header X-Forward-Proto http;
-        proxy_set_header X-Nginx-Proxy true;
+           proxy_set_header X-Nginx-Proxy true;
-        proxy_redirect off;
+           proxy_redirect off;
+        }
+        location ${REMOTE_LOCATION}websocket-tunnel {
+          proxy_pass ${REMOTE_PROXY_PASS}websocket-tunnel;
+          proxy_http_version 1.1;
+          proxy_set_header Upgrade ${DOLLAR}http_upgrade;
+          proxy_set_header Connection "Upgrade";
+          proxy_set_header Host ${DOLLAR}host;
+          proxy_set_header X-Real-IP ${DOLLAR}remote_addr;
+           proxy_set_header X-Forwarded-For ${DOLLAR}proxy_add_x_forwarded_for;
+           proxy_set_header X-Forward-Proto http;
+           proxy_set_header X-Nginx-Proxy true;
+           proxy_redirect off;
 
+        }
+                location ${REMOTE_LOCATION}websocket-tunnel/ {
+          proxy_pass ${REMOTE_PROXY_PASS}websocket-tunnel/;
+          proxy_http_version 1.1;
+          proxy_set_header Upgrade ${DOLLAR}http_upgrade;
+          proxy_set_header Connection "Upgrade";
+          proxy_set_header Host ${DOLLAR}host;
+          proxy_set_header X-Real-IP ${DOLLAR}remote_addr;
+           proxy_set_header X-Forwarded-For ${DOLLAR}proxy_add_x_forwarded_for;
+           proxy_set_header X-Forward-Proto http;
+           proxy_set_header X-Nginx-Proxy true;
+           proxy_redirect off;
+
+        }
+
+        location ${REMOTE_LOCATION} {
+           proxy_pass ${REMOTE_PROXY_PASS};
         }
     }
 

letsencrypt-nginx/website/index.html

@@ -1,22 +1,27 @@
 <h1>SiGyl Ltd</h1>
 
+Bespoke software development.
+
 <ul>
   <li>
-    <a href='https://sigyl.com/git'>repos</a>
+    <a href='https://sigyl.com/git'>git</a>
-  </li>
-  <li>
-    <a href='https://sigyl.com/chat'>chat</a>
   </li>
   <li>
     <a href='https://sigyl.com:5000'>drone</a>
   </li>
+  <li>
+    <a href='https://sigyl.com/remote'>remote</a>
+  </li>
+  <li>
+    <a href='https://sigyl.com/chat'>chat</a>
+  </li>
 </ul>
 
 <h2>home</h2>
 
 <ul>
   <li>
-    <a href='https://git.sigyl.com'>repos</a>
+    <a href='https://git.sigyl.com'>git</a>
   </li>
   <li>
     <a href='https://drone.sigyl.com'>drone</a>