This commit is contained in:
Giles Bradshaw 2020-05-10 17:16:11 +01:00
parent 9f390751a6
commit 0b8f98116b
37 changed files with 850 additions and 0 deletions

14
.drone-home.star Normal file
View File

@ -0,0 +1,14 @@
load("@this//drone:drone.star", "drone")
load("@this//drone:stack-name.star", "stackName")
load("@this//drone:stack-root.star", "stackRoot")
def main(ctx):
return drone(
ctx,
"home-deploy",
stackRoot,
stackName,
[]
)

79
docker-compose.yml Normal file
View File

@ -0,0 +1,79 @@
version: "3.7"
services:
drone-server:
# drone server application
deploy:
placement:
constraints: [node.labels.com.sigyl.git-stack == yes]
replicas: 1
restart_policy:
condition: any
image: drone/drone:1.7.0
volumes:
- drone:/var/lib/drone
- drone-data:/data
environment:
- DRONE_LOGS_DEBUG=true
- DRONE_LOGS_PRETTY=true
- DRONE_GITEA_SERVER=${DRONE_GITEA_SERVER}
- DRONE_GITEA_CLIENT_ID=${DRONE_GITEA_CLIENT_ID}
- DRONE_GITEA_CLIENT_SECRET=${DRONE_GITEA_CLIENT_SECRET}
- DRONE_SERVER_HOST=${DRONE_SERVER_HOST} # tunnel hostname
- DRONE_ADMIN=giles
- DRONE_SERVER_PROTO=https # tunnel adds https on top
- DRONE_SERVER_PORT=:8080
- DRONE_RPC_SECRET=${DRONE_RPC_SECRET}
- DRONE_USER_CREATE=username:giles,admin:true
- DRONE_AGENTS_ENABLED=true
- DRONE_CONVERT_PLUGIN_ENDPOINT=http://drone-starlark:3000
- DRONE_CONVERT_PLUGIN_SECRET=${DRONE_CONVERT_SECRET}
networks:
- appnet
- externalnet
drone-docker-runner:
# drone runner performs builds
deploy:
placement:
constraints: [node.labels.com.sigyl.git-stack == yes]
replicas: 1
restart_policy:
condition: any
image: drone/drone-runner-docker:1
volumes:
- /var/run/docker.sock:/var/run/docker.sock
environment:
- DRONE_RPC_PROTO=http
- DRONE_RPC_HOST=drone-server:8080
- DRONE_RPC_SECRET=${DRONE_RPC_SECRET}
- DRONE_RUNNER_CAPACITY=8
- DRONE_RUNNER_NAME="docker-runner"
networks:
- appnet
drone-starlark:
# drone starlark server converts starlark to yaml
deploy:
placement:
constraints: [node.labels.com.sigyl.git-stack == yes]
replicas: 1
restart_policy:
condition: any
image: ${LOCAL_DOCKER_REGISTRY}drone-starlark
environment:
- DRONE_DEBUG=true
- DRONE_SECRET=${DRONE_CONVERT_SECRET}
- DRONE_STARLARK_REPO_PATHS=this:/repos
- SIGYL_STACK_NAME=$SIGYL_STACK_NAME
- SIGYL_STACK_ROOT=$SIGYL_STACK_ROOT
networks:
- appnet
volumes:
drone:
drone-data:
networks:
appnet:
driver: overlay
#external: true
externalnet:
driver: overlay
external: true

View File

@ -0,0 +1,8 @@
FROM drone/drone-convert-starlark:1.1.0-beta.1
COPY repos /repos
COPY run.sh /
USER root
RUN apk update
RUN apk add gettext # enables envsubst
ENTRYPOINT []
CMD sh /run.sh

View File

@ -0,0 +1,31 @@
load("@this//:environment.star", "environment")
def buildDockerFolder(
dockerFile,
image,
tag,
folder,
name,
):
return {
"name": "build-{name}".format(
name = name,
),
"image": "docker:dind",
"volumes": [
{
"name": "dockersock",
"path": "/var/run",
},
],
"environment": environment([
"local-docker-registry",
]),
"commands": [
"cd {folder}".format(folder=folder),
"sh build-docker-folder.sh {dockerFile} {image} {tag}".format(
image = image,
dockerFile = dockerFile,
tag = tag,
),
],
}

View File

@ -0,0 +1,22 @@
load("@this//:environment.star", "environment")
def buildFolder(name, folder):
return {
"name": "build-{folder} {name}".format(
folder=folder,
name=name,
),
"image": "docker:dind",
"volumes": [
{
"name": "dockersock",
"path": "/var/run",
},
],
"environment": environment([
"local-docker-registry",
]),
"commands": [
"cd {folder}".format(folder=folder),
"sh build.sh {name} $${{LOCAL_DOCKER_REGISTRY}}".format(name = name),
],
}

View File

@ -0,0 +1,21 @@
load("@this//:environment.star", "environment")
def build(name):
return {
"name": "build-{name}".format(name=name),
"image": "docker:dind",
"volumes": [
{
"name": "dockersock",
"path": "/var/run",
},
],
"environment": environment([
"local-docker-registry",
]),
"commands": [
"cd {name}".format(name=name),
"docker build . -t $${{LOCAL_DOCKER_REGISTRY}}{name}".format(name=name),
"docker push $${{LOCAL_DOCKER_REGISTRY}}{name}".format(name=name),
],
}

View File

@ -0,0 +1,16 @@
load("@this//:from-secret.star", "fromSecret")
def clear(folder):
return {
"name": "clear",
"image": "appleboy/drone-ssh",
"settings": {
"host": fromSecret("ssh-host"),
"port": fromSecret("ssh-port"),
"username": fromSecret("ssh-user"),
"password": fromSecret("ssh-password"),
"script": [
"rm -r -f {folder}".format(folder = folder),
]
}
}

View File

@ -0,0 +1,38 @@
load("@this//:from-secret.star", "fromSecret")
load("@this//:map.star", "map")
load("@this//:environment.star", "environment")
load("@this//:export.star", "export")
def deploy(
filename,
name,
folder,
secrets,
commands,
ctx
):
return {
"name": "deploy {name}".format(name = name),
"image": "appleboy/drone-ssh",
"environment": environment(secrets),
"settings": {
"envs": [x.replace("-", "_") for x in secrets ],
"host": fromSecret("ssh-host"),
"port": fromSecret("ssh-port"),
"username": fromSecret("ssh-root-user"),
"password": fromSecret("ssh-root-password"),
"script": [
"set -e"
] +
map(export, secrets) +
[
"export DRONE_REPO_LINK=$${{DRONE_GITEA_SERVER}}/{namespace}/{name}".format(name=ctx.repo.name, namespace=ctx.repo.namespace),
"export DRONE_COMMIT={commit}".format(commit=ctx.build.commit),
"docker network prune -f",
"cd {folder}".format(folder=folder),
"docker stack rm {name}".format(name = name),
"sleep 30",
"docker stack deploy -c {filename} {name}".format(name= name, filename = filename),
] + commands
}
}

View File

@ -0,0 +1,81 @@
load("@this//:from-secret.star", "fromSecret")
load("@this//:print-secrets.star", "printSecrets")
load("@this//:map.star", "map")
load("@this//:environment.star", "environment")
load("@this//:echo.star", "echo")
load("@this//:export.star", "export")
load("@this//:echo-secret.star", "echoSecret")
load("@this//:wait.star", "wait")
load("@this//:build.star", "build")
load("@this//:scp.star", "scp")
load("@this//drone:public-secrets.star", "publicSecrets")
load("@this//drone:secret-secrets.star", "secretSecrets")
load("@this//:rescale.star", "rescale")
load("@this//:pull.star", "pull")
load("@this//:deploy.star", "deploy")
load("@this//:build-folder.star", "buildFolder")
load("@this//:build-docker-folder.star", "buildDockerFolder")
load("@this//:pipeline.star", "pipeline")
def drone(
ctx,
branch,
base,
name,
commands,
):
if ctx.build.branch == branch:
return [
pipeline(
branch,
[
wait(15, "wait"),
build("drone-starlark"),
printSecrets(
"env-drone",
publicSecrets,
secretSecrets,
),
scp(base),
pull(
"pull images",
[
"drone-starlark",
],
),
deploy(
"docker-compose.yml",
name,
base,
publicSecrets + secretSecrets,
commands,
ctx
),
],
[],
[
{
"name": "ca",
"host": {
"path": "/etc/docker/certs.d",
},
}
],
[
{
"name": "ca",
"path": "/etc/docker/certs.d",
},
]
),
]
else:
return pipeline(
ctx.build.branch,
[],
[],
[],
[],
)

View File

@ -0,0 +1,34 @@
publicSecrets = [
"title",
"description",
"certbot-email",
"drone-domain",
"drone-gitea-client-id",
"drone-gitea-server",
"drone-server-host",
"git-domain",
"local-docker-registry",
"ssh-host",
"guacamole-postgres-db",
"guacamole-postgres-user",
"sigyl-stack-root",
"sigyl-stack-name",
"ghost-mail-service",
"ghost-mail-user",
"chat-admin-name",
"chat-admin-email",
"gitea-mailer-host",
"gitea-mailer-from",
"gitea-mailer-user",
"gitea-app-name",
"commento-origin",
"commento-smtp-host",
"commento-smtp-port",
"commento-smtp-username",
"commento-smtp-from-address",
"commento-forbid-new-owners",
"commento-postgres-db",
"commento-postgres-user",
"commento-github-key",
"nagios-admin-user",
]

View File

@ -0,0 +1,24 @@
secretSecrets = [
"drone-convert-secret",
"drone-gitea-client-secret",
"drone-rpc-secret",
"guacamole-postgres-password",
"ngrok-auth-token",
"ghost-mail-password",
"ghost-mysql-root-password",
"chat-admin-password",
"gitea-server-lfs-jwt-secret",
"gitea-security-secret-key",
"gitea-security-internal-token",
"gitea-oauth2-jwt-secret",
"gitea-mailer-passwd",
"commento-smtp-password",
"commento-askimet-key",
"commento-postgres-password",
"commento-github-secret",
"matomo-mysql-root-password",
"matomo-mysql-password",
"nagios-admin-password",
"zabbix-mysql-root-password",
"zabbix-mysql-password",
]

View File

@ -0,0 +1 @@
stackName='drone'

View File

@ -0,0 +1 @@
stackRoot='/stack/drone'

View File

@ -0,0 +1,7 @@
load("@this//:secret-to-environment.star", "secretToEnvironment")
def echoSecret(secret):
return 'echo "export {environment}=???? ${environment}" >> ***filename*** # {secret}'.format(
secret = secret,
environment = secretToEnvironment(secret),
)

View File

@ -0,0 +1,7 @@
load("@this//:secret-to-environment.star", "secretToEnvironment")
def echo(secret):
return 'echo "export {environment}=\'${environment}\'" >> ***filename*** # {secret}'.format(
secret = secret,
environment = secretToEnvironment(secret),
)

View File

@ -0,0 +1,5 @@
load("@this//:from-secret.star", "fromSecret")
def environment(env):
return dict(
[(x.replace("-", "_").upper(), fromSecret(x)) for x in env]
)

View File

@ -0,0 +1,6 @@
load("@this//:secret-to-environment.star", "secretToEnvironment")
def export(secret):
return "export {toCaps}=${toCaps}".format(
toCaps = secretToEnvironment(secret),
)

View File

@ -0,0 +1,4 @@
def fromSecret(name):
return {
"from_secret": name
}

View File

@ -0,0 +1,2 @@
def map(fn, l):
return [fn(x) for x in l]

View File

@ -0,0 +1,32 @@
def pipeline(
name,
steps,
dependsOn,
volumes,
dockerVolumes
):
return {
"kind": "pipeline",
"name": name,
"depends_on": dependsOn,
"steps": steps,
"services": [
{
"name": "docker",
"image": "docker:dind",
"privileged": True,
"volumes": [
{
"name": "dockersock",
"path": "/var/run",
},
] + dockerVolumes,
}
],
"volumes": [
{
"name": "dockersock",
"temp": {},
},
] + volumes,
}

View File

@ -0,0 +1,24 @@
load("@this//:map.star", "map")
load("@this//:from-secret.star", "fromSecret")
load("@this//:environment.star", "environment")
load("@this//:echo.star", "echo")
load("@this//:export.star", "export")
load("@this//:echo-secret.star", "echoSecret")
def printSecrets(filename, env, secretEnv):
return {
"name": "print secrets",
"image": "appleboy/drone-ssh",
"environment": environment(env + secretEnv),
"settings": {
"envs": [x.replace("-", "_") for x in env + secretEnv ],
"host": fromSecret("ssh-host"),
"port": fromSecret("ssh-port"),
"username": fromSecret("ssh-user"),
"password": fromSecret("ssh-password"),
"script": [x.replace("***filename***", filename) for x in [
"rm -f env-stack",
] + map(echo, env)
+ map(echo, secretEnv)]
}
}

View File

@ -0,0 +1,98 @@
load("@this//:from-secret.star", "fromSecret")
load("@this//:print-secrets.star", "printSecrets")
load("@this//:map.star", "map")
load("@this//:environment.star", "environment")
load("@this//:echo.star", "echo")
load("@this//:export.star", "export")
load("@this//:echo-secret.star", "echoSecret")
load("@this//:wait.star", "wait")
load("@this//:build.star", "build")
load("@this//:scp.star", "scp")
load("@this//proxy:public-secrets.star", "publicSecrets")
load("@this//proxy:secret-secrets.star", "secretSecrets")
load("@this//:rescale.star", "rescale")
load("@this//:pull.star", "pull")
load("@this//:deploy.star", "deploy")
load("@this//:build-folder.star", "buildFolder")
load("@this//:build-docker-folder.star", "buildDockerFolder")
load("@this//:pipeline.star", "pipeline")
def drone(
ctx,
branch,
base,
name,
commands,
):
if ctx.build.branch == branch:
return [
pipeline(
branch,
[
wait(15, "wait"),
printSecrets(
"env-proxy",
publicSecrets,
secretSecrets,
),
build("ngrok-gitea"),
build("letsencrypt-nginx"),
buildDockerFolder(
"Dockerfile.git",
"$${LOCAL_DOCKER_REGISTRY}letsencrypt-nginx",
"$${LOCAL_DOCKER_REGISTRY}letsencrypt-git",
"letsencrypt-nginx",
"git",
),
buildDockerFolder(
"Dockerfile.drone",
"$${LOCAL_DOCKER_REGISTRY}letsencrypt-nginx",
"$${LOCAL_DOCKER_REGISTRY}letsencrypt-drone",
"letsencrypt-nginx",
"drone",
),
scp(base),
pull(
"pull images",
[
"ngrok-gitea",
"letsencrypt-git",
"letsencrypt-drone",
],
),
deploy(
"docker-compose.yml",
name,
base,
publicSecrets + secretSecrets,
commands,
ctx
),
],
[],
[
{
"name": "ca",
"host": {
"path": "/etc/docker/certs.d",
},
}
],
[
{
"name": "ca",
"path": "/etc/docker/certs.d",
},
]
),
]
else:
return pipeline(
ctx.build.branch,
[],
[],
[],
[],
)

View File

@ -0,0 +1,34 @@
publicSecrets = [
"title",
"description",
"certbot-email",
"drone-domain",
"drone-gitea-client-id",
"drone-gitea-server",
"drone-server-host",
"git-domain",
"local-docker-registry",
"ssh-host",
"guacamole-postgres-db",
"guacamole-postgres-user",
"sigyl-stack-root",
"sigyl-stack-name",
"ghost-mail-service",
"ghost-mail-user",
"chat-admin-name",
"chat-admin-email",
"gitea-mailer-host",
"gitea-mailer-from",
"gitea-mailer-user",
"gitea-app-name",
"commento-origin",
"commento-smtp-host",
"commento-smtp-port",
"commento-smtp-username",
"commento-smtp-from-address",
"commento-forbid-new-owners",
"commento-postgres-db",
"commento-postgres-user",
"commento-github-key",
"nagios-admin-user",
]

View File

@ -0,0 +1,24 @@
secretSecrets = [
"drone-convert-secret",
"drone-gitea-client-secret",
"drone-rpc-secret",
"guacamole-postgres-password",
"ngrok-auth-token",
"ghost-mail-password",
"ghost-mysql-root-password",
"chat-admin-password",
"gitea-server-lfs-jwt-secret",
"gitea-security-secret-key",
"gitea-security-internal-token",
"gitea-oauth2-jwt-secret",
"gitea-mailer-passwd",
"commento-smtp-password",
"commento-askimet-key",
"commento-postgres-password",
"commento-github-secret",
"matomo-mysql-root-password",
"matomo-mysql-password",
"nagios-admin-password",
"zabbix-mysql-root-password",
"zabbix-mysql-password",
]

View File

@ -0,0 +1 @@
stackName='proxy'

View File

@ -0,0 +1 @@
stackRoot='/stack/proxy'

View File

@ -0,0 +1,27 @@
load("@this//:from-secret.star", "fromSecret")
load("@this//:map.star", "map")
load("@this//:environment.star", "environment")
load("@this//:export.star", "export")
def pull(
name,
images,
):
secrets = [ "local-docker-registry"]
return {
"name": name,
"image": "appleboy/drone-ssh",
"environment": environment(secrets),
"settings": {
"envs": [x.replace("-", "_") for x in secrets ],
"host": fromSecret("ssh-host"),
"port": fromSecret("ssh-port"),
"username": fromSecret("ssh-root-user"),
"password": fromSecret("ssh-root-password"),
"script": [
"set -e"
] +
map(export, secrets) +
["docker pull $${{LOCAL_DOCKER_REGISTRY}}{image}".format(image=image) for image in images ]
}
}

View File

@ -0,0 +1,21 @@
load("@this//:from-secret.star", "fromSecret")
def rescale(
service,
scaleTo
):
return {
"name": "rescale {service}".format(service=service),
"image": "appleboy/drone-ssh",
"settings": {
"host": fromSecret("ssh-host"),
"port": fromSecret("ssh-port"),
"username": fromSecret("ssh-root-user"),
"password": fromSecret("ssh-root-password"),
"script": [
"set -e",
"docker service scale {service}=0".format(service=service),
"docker service scale {service}={scaleTo}".format(service=service, scaleTo=scaleTo),
]
}
}

View File

@ -0,0 +1,25 @@
def scp(target):
return {
"name": "scp files",
"image": "appleboy/drone-scp",
"settings": {
"host": {
"from_secret": "ssh-host",
},
"username": {
"from_secret": "ssh-user",
},
"password": {
"from_secret": "ssh-password",
},
"port": {
"from_secret": "ssh-port",
},
"command_timeout": "2m",
"target": target,
"source": [
".",
],
},
}

View File

@ -0,0 +1,2 @@
def secretToEnvironment(secret):
return secret.replace("-", "_").upper()

View File

@ -0,0 +1,87 @@
load("@this//:from-secret.star", "fromSecret")
load("@this//:print-secrets.star", "printSecrets")
load("@this//:map.star", "map")
load("@this//:environment.star", "environment")
load("@this//:echo.star", "echo")
load("@this//:export.star", "export")
load("@this//:echo-secret.star", "echoSecret")
load("@this//:wait.star", "wait")
load("@this//:build.star", "build")
load("@this//:scp.star", "scp")
load("@this//stack:public-secrets.star", "publicSecrets")
load("@this//stack:secret-secrets.star", "secretSecrets")
load("@this//:rescale.star", "rescale")
load("@this//:pull.star", "pull")
load("@this//:deploy.star", "deploy")
load("@this//:build-folder.star", "buildFolder")
load("@this//:build-docker-folder.star", "buildDockerFolder")
load("@this//:pipeline.star", "pipeline")
def drone(
ctx,
branch,
base,
name,
commands,
):
if ctx.build.branch == branch:
return [
pipeline(
branch,
[
wait(15, "wait"),
build("drone-starlark"),
printSecrets(
"env-stack",
publicSecrets,
secretSecrets,
),
build("gitea"),
build("guacamole-postgresql"),
build("ghost"),
scp(base),
pull(
"pull images",
[
"drone-starlark",
"gitea",
"ghost",
"guacamole-postgresql",
],
),
deploy(
"docker-compose.yml",
name,
base,
publicSecrets + secretSecrets,
commands,
ctx
),
],
[],
[
{
"name": "ca",
"host": {
"path": "/etc/docker/certs.d",
},
}
],
[
{
"name": "ca",
"path": "/etc/docker/certs.d",
},
]
),
]
else:
return pipeline(
ctx.build.branch,
[],
[],
[],
[],
)

View File

@ -0,0 +1,34 @@
publicSecrets = [
"title",
"description",
"certbot-email",
"drone-domain",
"drone-gitea-client-id",
"drone-gitea-server",
"drone-server-host",
"git-domain",
"local-docker-registry",
"ssh-host",
"guacamole-postgres-db",
"guacamole-postgres-user",
"sigyl-stack-root",
"sigyl-stack-name",
"ghost-mail-service",
"ghost-mail-user",
"chat-admin-name",
"chat-admin-email",
"gitea-mailer-host",
"gitea-mailer-from",
"gitea-mailer-user",
"gitea-app-name",
"commento-origin",
"commento-smtp-host",
"commento-smtp-port",
"commento-smtp-username",
"commento-smtp-from-address",
"commento-forbid-new-owners",
"commento-postgres-db",
"commento-postgres-user",
"commento-github-key",
"nagios-admin-user",
]

View File

@ -0,0 +1,24 @@
secretSecrets = [
"drone-convert-secret",
"drone-gitea-client-secret",
"drone-rpc-secret",
"guacamole-postgres-password",
"ngrok-auth-token",
"ghost-mail-password",
"ghost-mysql-root-password",
"chat-admin-password",
"gitea-server-lfs-jwt-secret",
"gitea-security-secret-key",
"gitea-security-internal-token",
"gitea-oauth2-jwt-secret",
"gitea-mailer-passwd",
"commento-smtp-password",
"commento-askimet-key",
"commento-postgres-password",
"commento-github-secret",
"matomo-mysql-root-password",
"matomo-mysql-password",
"nagios-admin-password",
"zabbix-mysql-root-password",
"zabbix-mysql-password",
]

View File

@ -0,0 +1 @@
stackName='${SIGYL_STACK_NAME}'

View File

@ -0,0 +1 @@
stackRoot='${SIGYL_STACK_ROOT}'

View File

@ -0,0 +1,8 @@
def wait(delay, name):
return {
"name": name,
"image": "alpine",
"commands": [
"sleep {delay}".format(delay = delay),
],
}

5
drone-starlark/run.sh Normal file
View File

@ -0,0 +1,5 @@
envsubst < /repos/stack/stack-name._star > /repos/stack/stack-name.star
envsubst < /repos/stack/stack-root._star > /repos/stack/stack-root.star
envsubst < /repos/stack/stack-name._star > /repos/proxy/stack-name.star
envsubst < /repos/stack/stack-root._star > /repos/proxy/stack-root.star
/bin/drone-convert-starlark