diff --git a/.drone/drone-home.jsonnet b/.drone/drone-home.jsonnet index b8459d7..da58df5 100644 --- a/.drone/drone-home.jsonnet +++ b/.drone/drone-home.jsonnet @@ -3,6 +3,7 @@ local environment = import 'node_modules/@sigyl/jsonnet-drone-environment/enviro local compose = import 'node_modules/@sigyl/jsonnet-compose/compose.libsonnet'; local secretSecrets = import 'lib/secret-secrets.libsonnet'; local publicSecrets = import 'lib/public-secrets.libsonnet'; +local util = import 'lib/util.libsonnet'; [ { kind: 'pipeline', @@ -12,11 +13,11 @@ local publicSecrets = import 'lib/public-secrets.libsonnet'; disable: false, depth: 0, }, - trigger: { + /*trigger: { event: [ 'tag', ], - }, + },*/ services: [ images.docker { privileged: true, @@ -45,10 +46,51 @@ local publicSecrets = import 'lib/public-secrets.libsonnet'; }, ], steps:[ + compose( + std.map( + function(secret) util.printEnv('env-squid', secret), + publicSecrets, + ) + ) + ( + images.ssh { + settings +: { + script: [ + 'rm -f env-squid', + ], + }, + }, + ) { + name: 'print env', + }, images.scp( '/stack/squid' ), images.wait(15), + images.docker { + name +: 'build docker:dind image:', + environment +: environment.environmentSecrets([ + 'LOCAL_DOCKER_REGISTRY', + 'LOCAL_REGISTRY_PASSWORD', + 'CA_CRT' + ]), + volumes: [ + { + name: 'dockersock', + path: '/var/run', + }, + ], + commands: [ + 'set -e', + 'sleep 15', + 'docker login $${LOCAL_DOCKER_REGISTRY} --username client --password "$${LOCAL_REGISTRY_PASSWORD}"', + 'cd docker-dind', + 'echo "$${CA_CRT}" > CA_crt.crt', + 'docker build . -t $${LOCAL_DOCKER_REGISTRY}docker:dind', + 'docker push $${LOCAL_DOCKER_REGISTRY}docker:dind', + 'docker logout $${LOCAL_DOCKER_REGISTRY}', + ], + }, /* images.docker { name +: 'build docker image:', environment +: environment.environmentSecrets([ @@ -71,32 +113,38 @@ local publicSecrets = import 'lib/public-secrets.libsonnet'; 'docker push $${LOCAL_DOCKER_REGISTRY}squid', 'docker logout $${LOCAL_DOCKER_REGISTRY}', ], - }, + } */ compose([ environment.envSet('local-docker-registry'), environment.envSet('local-registry-password'), + environment.envSet('ca-crt'), + environment.envSet('ca-key'), ])( images.ssh { name: 'deploy squid', settings +: { script +: [ + 'rm -f -R /stack/squid/.secrets', + 'mkdir -p /stack/squid/.secrets', + 'echo "$${CA_CRT}" > /stack/squid/.secrets/ca.crt', + 'echo "$${CA_KEY}" > /stack/squid/.secrets/ca.key', 'set -e', - "docker network prune -f", + //"docker network prune -f", "cd /stack/squid/myCA", - 'openssl genrsa -out CA_key.pem 2048', - 'openssl req -x509 -days 600 -new -nodes -key CA_key.pem -out CA_crt.pem -extensions v3_ca -config openssl.cnf -subj "/C=US/ST=California/L=Mountain View/O=Google/OU=Enterprise/CN=MyCA"', + //'openssl genrsa -out CA_key.pem 2048', + //'openssl req -x509 -days 600 -new -nodes -key CA_key.pem -out CA_crt.pem -extensions v3_ca -config openssl.cnf -subj "/C=US/ST=California/L=Mountain View/O=Google/OU=Enterprise/CN=MyCA"', 'cd ..', - "docker stack rm squid", - "sleep 60", - "docker volume rm squid_squid-cache", - 'export SQUID_IMAGE=$${LOCAL_DOCKER_REGISTRY}squid', - 'docker login $${LOCAL_DOCKER_REGISTRY} --username client --password "$${LOCAL_REGISTRY_PASSWORD}"', - 'docker pull $${SQUID_IMAGE}', + //"docker stack rm squid", + //"sleep 60", + // "docker volume rm squid_squid-cache", "docker stack deploy -c docker-compose.yml squid", ] } }, ), ], + image_pull_secrets: [ + 'dockerconfigjson' + ] } ] diff --git a/.drone/drone-home.yml b/.drone/drone-home.yml index 1290dda..8c2e14b 100644 --- a/.drone/drone-home.yml +++ b/.drone/drone-home.yml @@ -8,6 +8,47 @@ platform: arch: amd64 steps: +- name: print env + image: appleboy/drone-ssh + settings: + envs: + - drone_tag + - drone_commit + - drone_build_number + - drone_repo_name + - drone_repo_namespace + - ssh_host + - ssh_user + - ssh_root_user + - local_docker_registry + - ca_crt + host: + from_secret: ssh-host + key: + from_secret: ssh-key + port: + from_secret: ssh-port + script: + - rm -f env-squid + - "echo \"export SSH_HOST='$${SSH_HOST}'\" >> env-squid # \"ssh-host\"" + - "echo \"export SSH_USER='$${SSH_USER}'\" >> env-squid # \"ssh-user\"" + - "echo \"export SSH_ROOT_USER='$${SSH_ROOT_USER}'\" >> env-squid # \"ssh-root-user\"" + - "echo \"export LOCAL_DOCKER_REGISTRY='$${LOCAL_DOCKER_REGISTRY}'\" >> env-squid # \"local-docker-registry\"" + - "echo \"export CA_CRT='$${CA_CRT}'\" >> env-squid # \"ca-crt\"" + username: + from_secret: ssh-user + environment: + CA_CRT: + from_secret: ca-crt + LOCAL_DOCKER_REGISTRY: + from_secret: local-docker-registry + SSH_HOST: + from_secret: ssh-host + SSH_ROOT_USER: + from_secret: ssh-root-user + SSH_USER: + from_secret: ssh-user + - name: scp image: appleboy/drone-scp settings: @@ -29,18 +70,20 @@ steps: commands: - sleep 15 -- name: "dockerbuild docker image:" +- name: "dockerbuild docker:dind image:" image: docker:dind commands: - set -e - - pwd - sleep 15 - - cd docker - docker login $${LOCAL_DOCKER_REGISTRY} --username client --password "$${LOCAL_REGISTRY_PASSWORD}" - - docker build . -t $${LOCAL_DOCKER_REGISTRY}squid - - docker push $${LOCAL_DOCKER_REGISTRY}squid + - cd docker-dind + - echo "$${CA_CRT}" > CA_crt.crt + - docker build . -t $${LOCAL_DOCKER_REGISTRY}docker:dind + - docker push $${LOCAL_DOCKER_REGISTRY}docker:dind - docker logout $${LOCAL_DOCKER_REGISTRY} environment: + CA_CRT: + from_secret: ca-crt LOCAL_DOCKER_REGISTRY: from_secret: local-docker-registry LOCAL_REGISTRY_PASSWORD: @@ -60,6 +103,8 @@ steps: - drone_repo_namespace - local_docker_registry - local_registry_password + - ca_crt + - ca_key host: from_secret: ssh-host key: @@ -67,22 +112,21 @@ steps: port: from_secret: ssh-port script: + - rm -f -R /stack/squid/.secrets + - mkdir -p /stack/squid/.secrets + - echo "$${CA_CRT}" > /stack/squid/.secrets/ca.crt + - echo "$${CA_KEY}" > /stack/squid/.secrets/ca.key - set -e - - docker network prune -f - cd /stack/squid/myCA - - openssl genrsa -out CA_key.pem 2048 - - openssl req -x509 -days 600 -new -nodes -key CA_key.pem -out CA_crt.pem -extensions v3_ca -config openssl.cnf -subj "/C=US/ST=California/L=Mountain View/O=Google/OU=Enterprise/CN=MyCA" - cd .. - - docker stack rm squid - - sleep 60 - - docker volume rm squid_squid-cache - - export SQUID_IMAGE=$${LOCAL_DOCKER_REGISTRY}squid - - docker login $${LOCAL_DOCKER_REGISTRY} --username client --password "$${LOCAL_REGISTRY_PASSWORD}" - - docker pull $${SQUID_IMAGE} - docker stack deploy -c docker-compose.yml squid username: from_secret: ssh-user environment: + CA_CRT: + from_secret: ca-crt + CA_KEY: + from_secret: ca-key LOCAL_DOCKER_REGISTRY: from_secret: local-docker-registry LOCAL_REGISTRY_PASSWORD: @@ -105,8 +149,7 @@ volumes: host: path: /etc/docker/certs.d -trigger: - event: - - tag +image_pull_secrets: +- dockerconfigjson ... diff --git a/.drone/lib/public-secrets.libsonnet b/.drone/lib/public-secrets.libsonnet index e7e223c..4901236 100644 --- a/.drone/lib/public-secrets.libsonnet +++ b/.drone/lib/public-secrets.libsonnet @@ -3,4 +3,5 @@ 'ssh-user', 'ssh-root-user', 'local-docker-registry', + 'ca-crt', ] diff --git a/.drone/lib/secret-secrets.libsonnet b/.drone/lib/secret-secrets.libsonnet index 760fb48..28025b7 100644 --- a/.drone/lib/secret-secrets.libsonnet +++ b/.drone/lib/secret-secrets.libsonnet @@ -2,4 +2,5 @@ 'ssh-password', 'ssh-key', 'local-registry-password', + 'ca-key', ] diff --git a/.gitignore b/.gitignore index 54a14b2..3c3629e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1 @@ -myCA/*.pem node_modules diff --git a/README.md b/README.md index b7e2942..3b9fc78 100644 --- a/README.md +++ b/README.md @@ -7,11 +7,12 @@ inspired by https://github.com/salrashid123/squid_proxy ## making a CA ```shell -cd myCA openssl genrsa -out CA_key.pem 2048 -openssl req -x509 -days 600 -new -nodes -key CA_key.pem -out CA_crt.pem -extensions v3_ca -config openssl.cnf -subj "/C=US/ST=California/L=Mountain View/O=Google/OU=Enterprise/CN=MyCA" +openssl req -x509 -days 600 -new -nodes -key CA_key.pem -out CA_crt.pem -extensions v3_ca -config openssl.cnf -subj "/C=UK/ST=Devon/L=Rose Ash/O=Google/OU=SiGyl/CN=Proxy-ca" ``` +then set secrets ca-crt and ca-key to the created files + ## releasing [see here](https://sigyl.com/releases/) \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 97251be..df9b681 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,23 +1,39 @@ version: "3.7" services: - squid: + squid-4: deploy: placement: constraints: [node.labels.com.sigyl.git-stack == yes] replicas: 1 restart_policy: condition: any - image: ${SQUID_IMAGE} + image: wrouesnel/docker-squid4 + environment: + - MITM_PROXY=yes + - HTTP_PORT=3128 + - MITM_CERT=/run/secrets/ca.crt + - MITM_KEY=/run/secrets/ca.key + - VISIBLE_HOSTNAME=git.local-domain + - > + EXTRA_CONFIG1=tls_outgoing_options + capath=/etc/ssl/certs + options=NO_SSLv3,NO_TLSv1 min-version=1.2 + # - EXTRA_CONFIG2=sslproxy_cipher ECDHE+ECDSA+AESGCM:ECDHE+RSA+AESGCM:EECDH+ECDSA+AESGCM:EECDH+aRSA+AESGCM #:EECDH+ECDSA+SHA384:EECDH+ECDSA+SHA256:EECDH+aRSA+SHA384:EECDH+aRSA+SHA256:EECDH+aRSA+RC4:EECDH:EDH+aRSA:!RC4:!aNULL:!eNULL:!LOW:!3DES:!MD5:!EXP:!PSK:!SRP:!DSS + # these are basically to make everything canched + - 'EXTRA_CONFIG2=refresh_pattern ^http: 999999999 1000000000% 999999999 override-expire' + - 'EXTRA_CONFIG3=refresh_pattern ^https: 999999999 1000000000% 999999999 override-expire' + - EXTRA_CONFIG4= acl no_cache_domains dstdomain auth.docker.io + - EXTRA_CONFIG5=cache deny no_cache_domains volumes: - - squid-cache:/apps/squid/var/cache/squid - #- ./squid.intercept.conf:/etc/squid/squid.conf - - ./myCA/CA_crt.pem:/apps/CA_crt.pem - - ./myCA/CA_key.pem:/apps/CA_key.pem + - squid-4-cache:/var/cache/squid4 ports: - 3128:3128 networks: - appnet - externalnet + secrets: + - ca.crt + - ca.key squid-deb: deploy: placement: @@ -34,7 +50,7 @@ services: - appnet - externalnet volumes: - squid-cache: + squid-4-cache: squid-deb-cache: networks: @@ -43,3 +59,9 @@ networks: externalnet: driver: overlay external: true + +secrets: + 'ca.crt': + file: .secrets/ca.crt + 'ca.key': + file: .secrets/ca.key diff --git a/docker-dind/Dockerfile b/docker-dind/Dockerfile new file mode 100644 index 0000000..6fa9af8 --- /dev/null +++ b/docker-dind/Dockerfile @@ -0,0 +1,3 @@ +FROM docker:18.06.0-dind +COPY CA_crt.crt /usr/local/share/ca-certificates/CA_crt.crt +RUN update-ca-certificates diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index ebe0a65..0000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -FROM debian:8 -RUN apt-get -y update -RUN apt-get install -y curl supervisor git openssl build-essential libssl-dev wget vim curl -RUN mkdir -p /var/log/supervisor -WORKDIR /apps/ -RUN wget -O - http://www.squid-cache.org/Versions/v3/3.5/squid-3.5.27.tar.gz | tar zxfv - \ - && CPU=$(( `nproc --all`-1 )) \ - && cd /apps/squid-3.5.27/ \ - && ./configure --prefix=/apps/squid --enable-icap-client --enable-ssl --with-openssl --enable-ssl-crtd --enable-auth --enable-basic-auth-helpers="NCSA" \ - && make -j$CPU \ - && make install \ - && cd /apps \ - && rm -rf /apps/squid-3.5.27 -ADD . /apps/ - -RUN chown -R nobody:nogroup /apps/ -RUN mkdir -p /apps/squid/var/lib/ -RUN /apps/squid/libexec/ssl_crtd -c -s /apps/squid/var/lib/ssl_db -M 4MB -RUN /apps/squid/sbin/squid -N -f /apps/squid.cache.conf -z -RUN chown -R nobody:nogroup /apps/ - -EXPOSE 3128 -ENTRYPOINT ["/apps/squid/sbin/squid", "-NsY", "-f"] -CMD ["/apps/squid.intercept.conf"] diff --git a/docker/README.md b/docker/README.md deleted file mode 100644 index 390f317..0000000 --- a/docker/README.md +++ /dev/null @@ -1,3 +0,0 @@ -I made dhparam.pem - - openssl dhparam -outform PEM -out dhparam.pem 2048 \ No newline at end of file diff --git a/docker/dhparam.pem b/docker/dhparam.pem deleted file mode 100644 index 91e78f7..0000000 --- a/docker/dhparam.pem +++ /dev/null @@ -1,8 +0,0 @@ ------BEGIN DH PARAMETERS----- -MIIBCAKCAQEAk5sKJOAoHj9bZCoUyN0pnYwjzS2vCZWcNOCGKVO+MuyVhbphVGez -UidUVK7OIFX5XUNfrHvxKeN2NkHHfOJXAYdVD/0Th6Ead+nh/xtBw9+ycRhmLR1F -tQY1Kbv23j8h+rJ0q5aiMnCEKevnbPBlV3ARK1oXjAHVuT08flGOcRLb3Qp+qLKQ -xX5WGQcFzVJf56MA/bl5bUbuo7e8O1eZYjdtzz+nvk8zaYqEhqrrPkJDPveGdVKu -FYB4vRfBuOHc/1K9+kwzfNsAYhj51Qs64KjukmpjxZPTVojvnKRqiavRmgBdMWiL -J8VStE1njcXhusk3jGJazeQ5EsJA9u41qwIBAg== ------END DH PARAMETERS----- diff --git a/docker/squid.cache.conf b/docker/squid.cache.conf deleted file mode 100644 index 1396189..0000000 --- a/docker/squid.cache.conf +++ /dev/null @@ -1,3 +0,0 @@ -cache_dir aufs /apps/squid/var/cache/squid 10000 16 256 - -coredump_dir /apps/squid/var/cache \ No newline at end of file diff --git a/docker/squid.intercept.conf b/docker/squid.intercept.conf deleted file mode 100644 index 06f5e2e..0000000 --- a/docker/squid.intercept.conf +++ /dev/null @@ -1,70 +0,0 @@ -always_direct allow all - -acl localhost src 127.0.0.1/32 -acl to_localhost dst 127.0.0.0/8 -acl localnet src 10.0.0.0/8 # RFC1918 possible internal network -acl localnet src 172.16.0.0/12 # RFC1918 possible internal network -acl localnet src 192.168.0.0/16 # RFC1918 possible internal network -acl SSL_ports port 443 -acl Safe_ports port 80 # http -acl Safe_ports port 21 # ftp -acl Safe_ports port 443 # https -acl Safe_ports port 70 # gopher -acl Safe_ports port 210 # wais -acl Safe_ports port 1025-65535 # unregistered ports -acl Safe_ports port 280 # http-mgmt -acl Safe_ports port 488 # gss-http -acl Safe_ports port 591 # filemaker -acl Safe_ports port 777 # multiling http -acl CONNECT method CONNECT - -http_access allow all -http_access allow manager localhost -http_access deny manager - -htcp_access allow localnet -htcp_access deny all - - -visible_hostname git.local-domain - -http_port 3128 ssl-bump generate-host-certificates=on cert=/apps/CA_crt.pem key=/apps/CA_key.pem options=NO_SSLv3 dhparams=/apps/dhparam.pem - -always_direct allow all -acl excluded_sites ssl::server_name .wellsfargo.com -ssl_bump splice excluded_sites -ssl_bump bump all - -sslproxy_cert_error deny all -sslcrtd_program /apps/squid/libexec/ssl_crtd -s /apps/squid/var/lib/ssl_db -M 4MB sslcrtd_children 8 startup=1 idle=1 - -icap_enable on -icap_preview_enable on -icap_preview_size 128 -icap_send_client_ip on - -adaptation_access url_check allow all - -access_log /apps/squid/var/logs/access.log squid - -# these are basically to make everything canched -refresh_pattern ^http: 999999999 1000000000% 999999999 ignore-no-cache override-expire ignore-reload -refresh_pattern ^https: 999999999 1000000000% 999999999 ignore-no-cache override-expire ignore-reload - -debug_options 11,2 22,10 - -refresh_pattern ^ftp: 1440 20% 10080 -refresh_pattern ^gopher: 1440 0% 1440 -refresh_pattern (cgi-bin|\?) 0 0% 0 -refresh_pattern . 0 20% 4320 - -icp_port 3130 - - -coredump_dir /apps/squid/var/cache - - -cache_mem 1000 MB - -maximum_object_size 4096 MB -cache_dir aufs /apps/squid/var/cache/squid 10000 16 256 diff --git a/myCA/openssl.cnf b/openssl.cnf similarity index 100% rename from myCA/openssl.cnf rename to openssl.cnf