From 3b24eabf9f9928d0a2f328fe39fe85f779b5117e Mon Sep 17 00:00:00 2001 From: Giles Bradshaw Date: Thu, 6 Aug 2020 10:51:41 +0100 Subject: [PATCH] test: squid:4 --- .drone/drone-home.jsonnet | 25 ++++++- .drone/drone-home.yml | 4 +- docker/Dockerfile | 4 +- squid-4/Dockerfile | 121 ++++++++++++++++++++++++++++++++++ squid-4/README.md | 1 + squid-4/squid.bsh | 134 ++++++++++++++++++++++++++++++++++++++ squid-4/squid.conf.p2 | 46 +++++++++++++ 7 files changed, 330 insertions(+), 5 deletions(-) create mode 100644 squid-4/Dockerfile create mode 100644 squid-4/README.md create mode 100644 squid-4/squid.bsh create mode 100644 squid-4/squid.conf.p2 diff --git a/.drone/drone-home.jsonnet b/.drone/drone-home.jsonnet index cc2a75c..2fd0ed5 100644 --- a/.drone/drone-home.jsonnet +++ b/.drone/drone-home.jsonnet @@ -49,6 +49,29 @@ local publicSecrets = import 'lib/public-secrets.libsonnet'; '/stack/squid' ), images.wait(15), + images.docker { + name +: 'build docker image:', + environment +: environment.environmentSecrets([ + 'LOCAL_DOCKER_REGISTRY', + 'LOCAL_REGISTRY_PASSWORD', + ]), + volumes: [ + { + name: 'dockersock', + path: '/var/run', + }, + ], + commands: [ + 'set -e', + 'pwd', + 'sleep 15', + 'cd squid-4', + 'docker login $${LOCAL_DOCKER_REGISTRY} --username client --password "$${LOCAL_REGISTRY_PASSWORD}"', + 'docker build . -t $${LOCAL_DOCKER_REGISTRY}squid-4', + 'docker push $${LOCAL_DOCKER_REGISTRY}squid', + 'docker logout $${LOCAL_DOCKER_REGISTRY}', + ], + },/* images.docker { name +: 'build docker image:', environment +: environment.environmentSecrets([ @@ -71,7 +94,7 @@ local publicSecrets = import 'lib/public-secrets.libsonnet'; 'docker push $${LOCAL_DOCKER_REGISTRY}squid', 'docker logout $${LOCAL_DOCKER_REGISTRY}', ], - } /* + } */ /* compose([ environment.envSet('local-docker-registry'), environment.envSet('local-registry-password'), diff --git a/.drone/drone-home.yml b/.drone/drone-home.yml index 8b487d2..7307127 100644 --- a/.drone/drone-home.yml +++ b/.drone/drone-home.yml @@ -35,9 +35,9 @@ steps: - set -e - pwd - sleep 15 - - cd docker + - cd squid-4 - docker login $${LOCAL_DOCKER_REGISTRY} --username client --password "$${LOCAL_REGISTRY_PASSWORD}" - - docker build . -t $${LOCAL_DOCKER_REGISTRY}squid + - docker build . -t $${LOCAL_DOCKER_REGISTRY}squid-4 - docker push $${LOCAL_DOCKER_REGISTRY}squid - docker logout $${LOCAL_DOCKER_REGISTRY} environment: diff --git a/docker/Dockerfile b/docker/Dockerfile index baf56ef..e4bde17 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ FROM debian:8 RUN apt-get -y update -RUN apt-get install -y curl supervisor git openssl build-essential libssl-dev wget vim curl -RUN mkdir -p /var/log/supervisor +RUN apt-get install -y curl git openssl build-essential libssl-dev wget vim curl +#RUN mkdir -p /var/log/supervisor WORKDIR /apps/ RUN wget -O - http://www.squid-cache.org/Versions/v4/squid-4.12.tar.gz | tar zxfv - \ && CPU=$(( `nproc --all`-1 )) \ diff --git a/squid-4/Dockerfile b/squid-4/Dockerfile new file mode 100644 index 0000000..d12ffd4 --- /dev/null +++ b/squid-4/Dockerfile @@ -0,0 +1,121 @@ +ARG DOCKER_PREFIX= + +FROM ${DOCKER_PREFIX}ubuntu:artful + +ARG TRUST_CERT= + +RUN if [ ! -z "$TRUST_CERT" ]; then \ + echo "$TRUST_CERT" > /usr/local/share/ca-certificates/build-trust.crt ; \ + update-ca-certificates ; \ + fi + +# Normalize apt sources +RUN cat /etc/apt/sources.list | grep -v '^#' | sed /^$/d | sort | uniq > sources.tmp.1 && \ + cat /etc/apt/sources.list | sed s/deb\ /deb-src\ /g | grep -v '^#' | sed /^$/d | sort | uniq > sources.tmp.2 && \ + cat sources.tmp.1 sources.tmp.2 > /etc/apt/sources.list && \ + rm -f sources.tmp.1 sources.tmp.2 + +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get build-dep -y squid && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y wget tar xz-utils libssl-dev + +ARG SQUID_VERSION=4.0.21 + +# TODO: verify the squid download with the signing key +RUN mkdir /src \ + && cd /src \ + && wget http://www.squid-cache.org/Versions/v4/squid-$SQUID_VERSION.tar.xz \ + && mkdir squid \ + && tar -C squid --strip-components=1 -xvf squid-$SQUID_VERSION.tar.xz + +RUN cd /src/squid && \ + ./configure \ + --prefix=/usr \ + --datadir=/usr/share/squid4 \ + --sysconfdir=/etc/squid4 \ + --localstatedir=/var \ + --mandir=/usr/share/man \ + --enable-inline \ + --enable-async-io=8 \ + --enable-storeio="ufs,aufs,diskd,rock" \ + --enable-removal-policies="lru,heap" \ + --enable-delay-pools \ + --enable-cache-digests \ + --enable-underscores \ + --enable-icap-client \ + --enable-follow-x-forwarded-for \ + --enable-auth-basic="DB,fake,getpwnam,LDAP,NCSA,NIS,PAM,POP3,RADIUS,SASL,SMB" \ + --enable-auth-digest="file,LDAP" \ + --enable-auth-negotiate="kerberos,wrapper" \ + --enable-auth-ntlm="fake" \ + --enable-external-acl-helpers="file_userip,kerberos_ldap_group,LDAP_group,session,SQL_session,unix_group,wbinfo_group" \ + --enable-url-rewrite-helpers="fake" \ + --enable-eui \ + --enable-esi \ + --enable-icmp \ + --enable-zph-qos \ + --with-openssl \ + --enable-ssl \ + --enable-ssl-crtd \ + --disable-translation \ + --with-swapdir=/var/spool/squid4 \ + --with-logdir=/var/log/squid4 \ + --with-pidfile=/var/run/squid4.pid \ + --with-filedescriptors=65536 \ + --with-large-files \ + --with-default-user=proxy \ + --disable-arch-native + +ARG CONCURRENCY=1 + +RUN cd /src/squid && \ + make -j$CONCURRENCY && \ + make install + +# Download p2cli dependency +RUN wget -O /usr/local/bin/p2 \ + https://github.com/wrouesnel/p2cli/releases/download/r1/p2 && \ + chmod +x /usr/local/bin/p2 + +# Clone and build proxychains-ng for SSL upstream proxying +ARG PROXYCHAINS_COMMITTISH=7a233fb1f05bcbf3d7f5c91658932261de1e13cb + +RUN apt-get install -y git + +RUN git clone https://github.com/rofl0r/proxychains-ng.git /src/proxychains-ng && \ + cd /src/proxychains-ng && \ + git checkout $PROXYCHAINS_COMMITTISH && \ + ./configure --prefix=/usr --sysconfdir=/etc && \ + make -j$CONCURRENCY && make install + +ARG URL_DOH=https://github.com/wrouesnel/dns-over-https-proxy/releases/download/v0.0.2/dns-over-https-proxy_v0.0.2_linux-amd64.tar.gz + +RUN wget -O /tmp/doh.tgz \ + $URL_DOH && \ + tar -xvvf /tmp/doh.tgz --strip-components=1 -C /usr/local/bin/ && \ + chmod +x /usr/local/bin/dns-over-https-proxy + +COPY squid.conf.p2 /squid.conf.p2 +COPY squid.bsh /squid.bsh + +# Configuration environment +ENV HTTP_PORT=3128 \ + ICP_PORT= \ + HTCP_PORT= \ + MITM_PROXY= \ + MITM_CERT= \ + MITM_KEY= \ + VISIBLE_HOSTNAME=docker-squid4 \ + MAX_CACHE_SIZE=40000 \ + MAX_OBJECT_SIZE="1536 MB" \ + MEM_CACHE_SIZE="128 MB" \ + DNS_OVER_HTTPS_LISTEN_ADDR="127.0.0.153:53" \ + DNS_OVER_HTTPS_SERVER="https://dns.google.com/resolve" \ + DNS_OVER_HTTPS_NO_FALLTHROUGH="" \ + DNS_OVER_HTTPS_FALLTHROUGH_STATUSES=NXDOMAIN \ + DNS_OVER_HTTPS_PREFIX_SERVER= \ + DNS_OVER_HTTPS_SUFFIX_SERVER= + +EXPOSE 3128 + +ENTRYPOINT [ "/squid.bsh" ] \ No newline at end of file diff --git a/squid-4/README.md b/squid-4/README.md new file mode 100644 index 0000000..7777e56 --- /dev/null +++ b/squid-4/README.md @@ -0,0 +1 @@ +from https://github.com/wrouesnel/docker-squid4 diff --git a/squid-4/squid.bsh b/squid-4/squid.bsh new file mode 100644 index 0000000..33e6e6e --- /dev/null +++ b/squid-4/squid.bsh @@ -0,0 +1,134 @@ +#!/bin/bash + +# Setup the ssl_cert directory +if [ ! -d /etc/squid4/ssl_cert ]; then + mkdir /etc/squid4/ssl_cert +fi + +chown -R proxy:proxy /etc/squid4 +chmod 700 /etc/squid4/ssl_cert + +# Setup the squid cache directory +if [ ! -d /var/cache/squid4 ]; then + mkdir -p /var/cache/squid4 +fi +chown -R proxy: /var/cache/squid4 +chmod -R 750 /var/cache/squid4 + +if [ ! -z $MITM_PROXY ]; then + if [ ! -z $MITM_KEY ]; then + echo "Copying $MITM_KEY as MITM key..." + cp $MITM_KEY /etc/squid4/ssl_cert/mitm.pem + chown root:proxy /etc/squid4/ssl_cert/mitm.pem + fi + + if [ ! -z $MITM_CERT ]; then + echo "Copying $MITM_CERT as MITM CA..." + cp $MITM_CERT /etc/squid4/ssl_cert/mitm.crt + chown root:proxy /etc/squid4/ssl_cert/mitm.crt + fi + + if [ -z $MITM_CERT ] || [ -z $MITM_KEY ]; then + echo "Must specify $MITM_CERT AND $MITM_KEY." 1>&2 + exit 1 + fi +fi + +chown proxy: /dev/stdout +chown proxy: /dev/stderr + +# Initialize the certificates database +/usr/libexec/security_file_certgen -c -s /var/spool/squid4/ssl_db +chown -R proxy: /var/spool/squid4/ssl_db + +#ssl_crtd -c -s +#ssl_db + +# Set the configuration +if [ "$CONFIG_DISABLE" != "yes" ]; then + p2 -t /squid.conf.p2 > /etc/squid4/squid.conf + + # Parse the cache peer lines from the environment and add them to the + # configuration + echo '# CACHE PEERS FROM DOCKER' >> /etc/squid4/squid.conf + env | grep 'CACHE_PEER' | sort | while read cacheline; do + echo "# $cacheline " >> /etc/squid4/squid.conf + line=$(echo $cacheline | cut -d'=' -f2-) + echo "cache_peer $line" >> /etc/squid4/squid.conf + done + + # Parse the extra config lines and append them to the configuration + echo '# EXTRA CONFIG FROM DOCKER' >> /etc/squid4/squid.conf + env | grep 'EXTRA_CONFIG' | sort | while read extraline; do + echo "# $extraline " >> /etc/squid4/squid.conf + line=$(echo $extraline | cut -d'=' -f2-) + echo "$line" >> /etc/squid4/squid.conf + done +else + echo "/etc/squid4/squid.conf: CONFIGURATION TEMPLATING IS DISABLED." +fi + +if [ "$DNS_OVER_HTTPS" = "yes" ]; then + echo "Starting DNS-over-HTTPS proxy..." + # TODO: find a way to tie this to the proxychains config + dns-over-https-proxy -default "$DNS_OVER_HTTPS_SERVER" \ + -address "$DNS_OVER_HTTPS_LISTEN_ADDR" \ + -primary-dns "$DNS_OVER_HTTPS_PREFIX_SERVER" \ + -fallback-dns "$DNS_OVER_HTTPS_SUFFIX_SERVER" \ + -no-fallthrough "$(echo $DNS_OVER_HTTPS_NO_FALLTHROUGH | tr -s ' ' ',')" \ + -fallthrough-statuses "$DNS_OVER_HTTPS_FALLTHROUGH_STATUSES" & + echo "Adding dns_nameservers line to squid.conf..." + echo "dns_nameservers $(echo $DNS_OVER_HTTPS_LISTEN_ADDR | cut -d':' -f1)" >> /etc/squid4/squid.conf +fi + +if [ ! -e /etc/squid4/squid.conf ]; then + echo "ERROR: /etc/squid4/squid.conf does not exist. Squid will not work." + exit 1 +fi + +# If proxychains is requested and config templating is active +if [ "$PROXYCHAIN" = "yes" ] && [ "$CONFIG_DISABLE" != "yes" ]; then + echo "# PROXYCHAIN CONFIG FROM DOCKER" > /etc/proxychains.conf + # Enable remote DNS proxy + if [ ! -z "$PROXYCHAIN_DNS" ]; then + echo "proxy_dns" >> /etc/proxychains.conf + fi + # Configure proxy type + if [ ! -z "$PROXYCHAIN_TYPE" ]; then + echo "$PROXYCHAIN_TYPE" >> /etc/proxychains.conf + else + echo "strict_chain" >> /etc/proxychains.conf + fi + + echo "[ProxyList]" >> /etc/proxychains.conf + env | grep 'PROXYCHAIN_PROXY' | sort | while read proxyline; do + echo "# $proxyline " >> /etc/squid4/squid.conf + line=$(echo $proxyline | cut -d'=' -f2-) + echo "$line" >> /etc/proxychains.conf + done +else + echo "/etc/proxychains.conf : CONFIGURATION TEMPLATING IS DISABLED" +fi + +# Build the configuration directories if needed +squid -z -N + +if [ "$PROXYCHAIN" = "yes" ]; then + if [ ! -e /etc/proxychains.conf ]; then + echo "ERROR: /etc/proxychains.conf does not exist. Squid with proxychains will not work." + exit 1 + fi + # Start squid with proxychains + proxychains4 -f /etc/proxychains.conf squid -N 2>&1 & + PID=$! +else + # Start squid normally + squid -N 2>&1 & + PID=$! +fi + +# This construct allows signals to kill the container successfully. +trap "kill -TERM $(jobs -p)" INT TERM +wait $PID +wait $PID +exit $? \ No newline at end of file diff --git a/squid-4/squid.conf.p2 b/squid-4/squid.conf.p2 new file mode 100644 index 0000000..fa6e8cb --- /dev/null +++ b/squid-4/squid.conf.p2 @@ -0,0 +1,46 @@ +# TEMPLATED CONFIGURATION FILE. UPDATED ON EACH RUN. + +# Default all logs to stdout and stderr +logfile_rotate 0 +access_log stdio:/dev/stdout combined +cache_store_log stdio:/dev/stdout +cache_log /dev/stderr +netdb_filename stdio:/var/cache/squid4/netdb.state + +# Visible hostname to allow multi-squid +visible_hostname {{VISIBLE_HOSTNAME|default:"docker-squid4"}} + +{% if DISABLE_CACHE|default:"" != "yes" %} +# Cache directory is fixed since we'll bind mount. +cache_dir aufs /var/cache/squid4 {{MAX_CACHE_SIZE|default:"40000"}} 16 256 +{% endif %} + +maximum_object_size {{MAX_OBJECT_SIZE|default:"1536 MB"}} +cache_mem {{MEM_CACHE_SIZE|default:"128 MB"}} + +tls_outgoing_options capath=/etc/ssl/certs \ + options={{TLS_OPTIONS|default:"NO_SSLv3,NO_TLSv1"}} \ + cipher=ALL:!RC4:!aNULL:!eNULL:!LOW:!3DES:!MD5:!EXP:!PSK:!SRP:!DSS + +http_port {{HTTP_PORT}} {% if MITM_PROXY|default:"" == "yes" %} ssl-bump \ + generate-host-certificates=on \ + dynamic_cert_mem_cache_size=4MB \ + cert=/etc/squid4/ssl_cert/mitm.crt \ + key=/etc/squid4/ssl_cert/mitm.pem +{% endif %} + +{% if MITM_PROXY|default:"" == "yes" %} +ssl_bump server-first all +{% endif %} + +{% if ICP_PORT|default:"" != "" %} +icp_port {{ICP_PORT}} +icp_access allow all +{% endif %} + +{% if HTCP_PORT|default:"" != "" %} +htcp_port {{HTCP_PORT}} +htcp_access allow all +{% endif %} + +http_access allow all \ No newline at end of file