split stack into services and proxy

mend
proxied services on external net
2020-08-11 · 2020-05-08 10:28:32 +01:00 · 2020-05-08 09:18:55 +01:00 · 2020-05-08 09:17:57 +01:00 · 2020-05-07 21:15:42 +01:00 · 2020-05-07 21:11:45 +01:00
4 changed files with 138 additions and 109 deletions
--- a/.drone-home.star
+++ b/.drone-home.star
@ -10,9 +10,9 @@ def main(ctx):
    stackRoot,
    stackName,
    [
-      "docker service scale {name}_ngrok=1".format(name=stackName),
+      "docker service scale proxy_ngrok=1".format(name=stackName),
      "docker service scale {name}_chat=1".format(name=stackName),
-      "docker service scale {name}_letsencrypt-drone=1".format(name=stackName),
+      "docker service scale proxy_letsencrypt-drone=1".format(name=stackName),
    ]
  )
  
--- a/docker-compose-proxy.yml
+++ b/docker-compose-proxy.yml
@ -0,0 +1,115 @@
+version: "3.7"
+services:
+  letsencrypt-git:
+    # nginx reverse proxy for all apps (except drone in a subdomain)
+    # automatically obtains and refreshes ssl certificates with letsencrypt
+    deploy:
+      placement:
+        constraints: [node.labels.com.sigyl.git-stack == yes]
+      replicas: 1
+      restart_policy:
+        condition: any
+    image: ${LOCAL_DOCKER_REGISTRY}letsencrypt-git
+    environment:
+      - CERTBOT_EMAIL=${CERTBOT_EMAIL}
+      - SERVER_NAME=${GIT_DOMAIN}
+      - GIT_PROXY_PASS=http://${SIGYL_STACK_NAME}_gitea:3000/
+      - BLOG_PROXY_PASS=http://${SIGYL_STACK_NAME}_ghost:2368/
+      - CHAT_PROXY_PASS=http://${SIGYL_STACK_NAME}_chat:3000/
+      - COMMENTO_PROXY_PASS=http://${SIGYL_STACK_NAME}_commento:8080/
+      - REMOTE_PROXY_PASS=http://${SIGYL_STACK_NAME}_guacamole:8080/guacamole/
+      - DRONE_PROXY_PASS=http://${SIGYL_STACK_NAME}_drone-server:8080/
+      - REGISTRY_PROXY_PASS=http://registry-1:5000
+      - PORTAINER_PROXY_PASS=http://${SIGYL_STACK_NAME}_portainer:9000/
+      - PORTAINER_LOCATION=/portainer/
+      - MATOMO_PROXY_PASS=http://${SIGYL_STACK_NAME}_matomo-web/
+      - MATOMO_LOCATION=/analytics/
+      - NAGIOS_PROXY_PASS=http://${SIGYL_STACK_NAME}_nagios/
+      - NAGIOS_LOCATION=/nagios/
+      - ZABBIX_PROXY_PASS=http://${SIGYL_STACK_NAME}_zabbix-web:8080/
+      - ZABBIX_LOCATION=/zabbix/
+      - GIT_LOCATION=/git/
+      - BLOG_LOCATION=/
+      - CHAT_LOCATION=/chat/
+      - COMMENTO_LOCATION=/comment/
+      - REMOTE_LOCATION=/remote/
+      - TITLE=$TITLE
+      - DESCRIPTION=$DESCRIPTION
+      - DRONE_REPO_LINK=$DRONE_REPO_LINK
+      - DRONE_COMMIT=$DRONE_COMMIT   
+    volumes:
+      - letsencrypt-git:/etc/letsencrypt
+    networks:
+      - externalnet
+      - appnet
+    ports:
+      - 80:80
+      - 443:443
+      - 5000:5000
+      - 5001:5001
+      - 5005:5005
+  letsencrypt-drone:
+    # reverse proxy for drone in a subdomain
+    deploy:
+      placement:
+        constraints: [node.labels.com.sigyl.git-stack-data == yes]
+      replicas: 0
+      restart_policy:
+        condition: any
+    image: ${LOCAL_DOCKER_REGISTRY}letsencrypt-drone
+    environment:
+      - CERTBOT_EMAIL=${CERTBOT_EMAIL}
+      - SERVER_NAME=${DRONE_DOMAIN}
+      - PROXY_PASS=http://${SIGYL_STACK_NAME}_drone-server:8080/
+    volumes:
+      - letsencrypt-drone:/etc/letsencrypt
+    networks:
+      - appnet
+      - externalnet
+  ngrok:
+    # ngrok tunnel client
+    deploy:
+      placement:
+        constraints: [node.labels.com.sigyl.git-stack == yes]
+      replicas: 0
+      restart_policy:
+        condition: any
+    image: ${LOCAL_DOCKER_REGISTRY}ngrok-gitea
+    ports:
+      - "4040:4040"
+    environment:
+      - GIT_DOMAIN=${GIT_DOMAIN}
+      - DRONE_DOMAIN=${DRONE_DOMAIN}
+      - REMOTE_DOMAIN=${REMOTE_DOMAIN}
+      - BLOG_DOMAIN=${BLOG_DOMAIN}
+      - CHAT_DOMAIN=${CHAT_DOMAIN}
+      - NGROK_AUTH_TOKEN=${NGROK_AUTH_TOKEN}
+    networks:
+      - appnet
+  registry-1:
+    deploy:
+      placement:
+        constraints: [node.labels.com.sigyl.git-stack == yes]
+      replicas: 1
+      restart_policy:
+        condition: any
+    image: registry:2
+    volumes:
+      - registry-data:/var/lib/registry
+    environment:
+      - REGISTRY_HTTP_ADDR=0.0.0.0:5000
+    networks:
+      - appnet
+      - externalnet
+volumes:
+  letsencrypt-git:
+  letsencrypt-drone:
+  registry-data:
+
+networks:
+  appnet:
+    driver: overlay
+    #external: true
+  externalnet:
+    driver: overlay
+    external: true
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -1,73 +1,6 @@
 version: "3.7"
 services:
-  letsencrypt-git:
-    # nginx reverse proxy for all apps (except drone in a subdomain)
-    # automatically obtains and refreshes ssl certificates with letsencrypt
-    deploy:
-      placement:
-        constraints: [node.labels.com.sigyl.git-stack == yes]
-      replicas: 1
-      restart_policy:
-        condition: any
-    image: ${LOCAL_DOCKER_REGISTRY}letsencrypt-git
-    environment:
-      - CERTBOT_EMAIL=${CERTBOT_EMAIL}
-      - SERVER_NAME=${GIT_DOMAIN}
-      - GIT_PROXY_PASS=http://gitea:3000/
-      - BLOG_PROXY_PASS=http://ghost:2368/
-      - CHAT_PROXY_PASS=http://chat:3000/
-      - COMMENTO_PROXY_PASS=http://commento:8080/
-      - REMOTE_PROXY_PASS=http://guacamole:8080/guacamole/
-      - DRONE_PROXY_PASS=http://drone-server:8080/
-      - REGISTRY_PROXY_PASS=http://registry-1:5000
-      - PORTAINER_PROXY_PASS=http://portainer:9000/
-      - PORTAINER_LOCATION=/portainer/
-      - MATOMO_PROXY_PASS=http://matomo-web/
-      - MATOMO_LOCATION=/analytics/
-      - NAGIOS_PROXY_PASS=http://nagios/
-      - NAGIOS_LOCATION=/nagios/
-      - ZABBIX_PROXY_PASS=http://zabbix-web:8080/
-      - ZABBIX_LOCATION=/zabbix/
-      - GIT_LOCATION=/git/
-      - BLOG_LOCATION=/
-      - CHAT_LOCATION=/chat/
-      - COMMENTO_LOCATION=/comment/
-      - REMOTE_LOCATION=/remote/
-      - DRONE_SERVER_HOST=$DRONE_SERVER_HOST
-      - TITLE=$TITLE
-      - DESCRIPTION=$DESCRIPTION
-      - DRONE_REPO_LINK=$DRONE_REPO_LINK
-      - DRONE_COMMIT=$DRONE_COMMIT
-      
-    volumes:
-      - letsencrypt-git:/etc/letsencrypt
-    networks:
-      - appnet
-    ports:
-      - 80:80
-      - 443:443
-      - 5000:5000
-      - 5001:5001
-      - 5005:5005
-  letsencrypt-drone:
-    # reverse proxy for drone in a subdomain
-    deploy:
-      placement:
-        constraints: [node.labels.com.sigyl.git-stack-data == yes]
-      replicas: 0
-      restart_policy:
-        condition: any
-    image: ${LOCAL_DOCKER_REGISTRY}letsencrypt-drone
-    environment:
-      - CERTBOT_EMAIL=${CERTBOT_EMAIL}
-      - SERVER_NAME=${DRONE_DOMAIN}
-      - PROXY_PASS=http://drone-server:8080/
-    volumes:
-      - letsencrypt-drone:/etc/letsencrypt
-    networks:
-      - appnet
  gitea:
-    # gitea application
    deploy:
      placement:
        constraints: [node.labels.com.sigyl.git-stack == yes]
@ -98,26 +31,7 @@ services:
      - 22:22
    networks:
      - appnet
-  ngrok:
-    # ngrok tunnel client
-    deploy:
-      placement:
-        constraints: [node.labels.com.sigyl.git-stack == yes]
-      replicas: 0
-      restart_policy:
-        condition: any
-    image: ${LOCAL_DOCKER_REGISTRY}ngrok-gitea
-    ports:
-      - "4040:4040"
-    environment:
-      - GIT_DOMAIN=${GIT_DOMAIN}
-      - DRONE_DOMAIN=${DRONE_DOMAIN}
-      - REMOTE_DOMAIN=${REMOTE_DOMAIN}
-      - BLOG_DOMAIN=${BLOG_DOMAIN}
-      - CHAT_DOMAIN=${CHAT_DOMAIN}
-      - NGROK_AUTH_TOKEN=${NGROK_AUTH_TOKEN}
-    networks:
-      - appnet
+      - externalnet
  drone-server:
    # drone server application
    deploy:
@ -147,6 +61,7 @@ services:
      - DRONE_CONVERT_PLUGIN_SECRET=${DRONE_CONVERT_SECRET}
    networks:
      - appnet
+      - externalnet
  drone-docker-runner:
    # drone runner performs builds
    deploy:
@ -159,11 +74,13 @@ services:
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
    environment:
-      - DRONE_RPC_PROTO=https
-      - DRONE_RPC_HOST=${DRONE_SERVER_HOST}
+      - DRONE_RPC_PROTO=http
+      - DRONE_RPC_HOST=drone-server:8080
      - DRONE_RPC_SECRET=${DRONE_RPC_SECRET}
      - DRONE_RUNNER_CAPACITY=8
      - DRONE_RUNNER_NAME="docker-runner"
+    networks:
+      - appnet
  drone-starlark:
    # drone starlark server converts starlark to yaml
    deploy:
@ -203,21 +120,6 @@ services:
      - registry-key
    ports:
      - 5003:5000
-  registry-1:
-    # internal registry #1 (why?)
-    deploy:
-      placement:
-        constraints: [node.labels.com.sigyl.git-stack == yes]
-      replicas: 1
-      restart_policy:
-        condition: any
-    image: registry:2
-    volumes:
-      - registry-data:/var/lib/registry
-    environment:
-      - REGISTRY_HTTP_ADDR=0.0.0.0:5000
-    networks:
-      - appnet
  registry-cache:
    # registry cache (used?)
    deploy:
@ -268,8 +170,10 @@ services:
      - database__connection__user=root
      - database__connection__password=$GHOST_MYSQL_ROOT_PASSWORD
      - database__connection__database=ghost
+      - database__pool__min=0 # https://github.com/knex/knex/issues/975
    networks:
      - appnet
+      - externalnet
  ghost-mysql:
    image: mysql:5.7
    deploy:
@ -308,6 +212,7 @@ services:
      COMMENTO_POSTGRES: postgres://${COMMENTO_POSTGRES_USER}:${COMMENTO_POSTGRES_PASSWORD}@commento-postgres:5432/${COMMENTO_POSTGRES_DB}?sslmode=disable
    networks:
      - appnet
+      - externalnet
  commento-postgres:
    deploy:
      placement:
@ -356,6 +261,7 @@ services:
      - ./nagios/contacts/contacts.cfg:/opt/nagios/etc/objects/contacts.cfg
    networks:
      - appnet
+      - externalnet
  matomo:
    image: matomo:fpm-alpine
    deploy:
@ -394,6 +300,7 @@ services:
      - ./matomo/matomo.conf:/etc/nginx/conf.d/default.conf:ro
    networks:
      - appnet
+      - externalnet
  matomo-mariadb:
    deploy:
      placement:
@ -459,6 +366,7 @@ services:
    image: zabbix/zabbix-web-nginx-mysql
    networks:
      - appnet
+      - externalnet
    environment:
      DB_SERVER_HOST: zabbix-mariadb
      MYSQL_USER: zabbix
@ -494,6 +402,7 @@ services:
      - GUACD_HOSTNAME=guacd
    networks:
      - appnet
+      - externalnet
  chat:
    deploy:
      placement:
@ -504,6 +413,7 @@ services:
    image: rocketchat/rocket.chat:3.0.7
    networks:
      - appnet
+      - externalnet
    environment:
      - MONGO_OPLOG_URL=mongodb://chat-mongo:27017/local
      - ROOT_URL=https://${GIT_DOMAIN}/chat
@ -546,6 +456,7 @@ services:
    networks:
      #- proxy
      - appnet
+      - externalnet
  portainer-agent:
    image: portainer/agent:1.5.1
    environment:
@ -571,8 +482,6 @@ volumes:
  registry-cache-data:
  guacamole-postgresql-data:
  commento-postgresql-data:
-  letsencrypt-git:
-  letsencrypt-drone:
  ghost-content:
  ghost-data:
  ghost-content-adapters:
@ -591,9 +500,11 @@ networks:
  appnet:
    driver: overlay
    #external: true
+  externalnet:
+    driver: overlay
+    external: true
 secrets:
  'registry-cert':
    file: .certificates/registry.crt
  'registry-key':
    file: .certificates/registry.key
-  
--- a/drone-starlark/repos/deploy.star
+++ b/drone-starlark/repos/deploy.star
@ -30,6 +30,9 @@ def deploy(
        "export DRONE_COMMIT={commit}".format(commit=ctx.build.commit),
        "docker network prune -f",
        "cd {folder}".format(folder=folder),
+        "docker stack rm proxy",
+        "sleep 30",
+        "docker stack deploy -c docker-compose-proxy.yml proxy",
        "docker stack rm {name}".format(name = name),
        "sleep 30",
        "docker stack deploy -c {filename} {name}".format(name= name, filename = filename),
Author	SHA1	Message	Date
Giles Bradshaw	07dfba6090	split stack into services and proxy continuous-integration/drone/push Build was killed Details	2020-05-08 10:28:32 +01:00
Giles Bradshaw	24b0700054	mend continuous-integration/drone/push Build was killed Details	2020-05-08 09:18:55 +01:00
Giles Bradshaw	8c536b9710	proxied services on external net	2020-05-08 09:17:57 +01:00
Giles Bradshaw	479439567d	. continuous-integration/drone/push Build was killed Details	2020-05-07 21:15:42 +01:00
Giles Bradshaw	fa23654cf4	matomo on externalnet continuous-integration/drone/push Build is passing Details	2020-05-07 21:11:45 +01:00
Giles Bradshaw	a14ddf7f3d	ghost database pool = 0 https://github.com/knex/knex/issues/975 continuous-integration/drone/push Build was killed Details	2020-05-07 16:28:07 +01:00
Giles Bradshaw	c40b34634e	drone server host on appnet continuous-integration/drone/push Build was killed Details	2020-05-07 15:25:16 +01:00
Giles Bradshaw	583aab097d	put drone-runner on appnet continuous-integration/drone/push Build was killed Details	2020-05-07 15:03:21 +01:00