From 310d522c8f50120d3330b0c33b31c62a09ee785b Mon Sep 17 00:00:00 2001 From: Micah Hausler Date: Fri, 17 Jun 2016 15:27:23 -0400 Subject: [PATCH 1/9] Added Confd to update configs --- Dockerfile | 53 +++++++++++++++++++------- LICENSE | 21 +++++++++++ README.md | 109 +++++++++++++++-------------------------------------- nginx.conf | 83 ++++++++++++++++++++++++++++++++++++++++ proxy.tmpl | 35 +++++++++++++++++ proxy.toml | 12 ++++++ 6 files changed, 220 insertions(+), 93 deletions(-) create mode 100644 LICENSE create mode 100644 nginx.conf create mode 100644 proxy.tmpl create mode 100644 proxy.toml diff --git a/Dockerfile b/Dockerfile index 3084041..488b6c7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,20 +2,45 @@ FROM debian:jessie WORKDIR /tmp -RUN apt-get -y update -RUN apt-get -y install curl build-essential libpcre3 libpcre3-dev zlib1g-dev libssl-dev git && \ - curl -LO http://nginx.org/download/nginx-1.9.3.tar.gz && \ - tar zxf nginx-1.9.3.tar.gz && \ - cd nginx-1.9.3 && \ - git clone https://github.com/anomalizer/ngx_aws_auth.git && \ - ./configure --with-http_ssl_module --add-module=ngx_aws_auth && \ - make install && \ - cd /tmp && \ - rm -f nginx-1.9.3.tar.gz && \ - rm -rf nginx-1.9.3 && \ - apt-get purge -y curl git && \ - apt-get autoremove -y +ENV NGINX_VERSION 1.11.1 +RUN apt-get -y update \ + && apt-get -y install \ + curl \ + build-essential \ + libpcre3 \ + libpcre3-dev \ + zlib1g-dev \ + libssl-dev \ + wget \ + git \ + && curl -LO http://nginx.org/download/nginx-$NGINX_VERSION.tar.gz \ + && tar zxf nginx-$NGINX_VERSION.tar.gz \ + && cd nginx-$NGINX_VERSION \ + && git clone https://github.com/anomalizer/ngx_aws_auth.git \ + && ./configure --with-http_ssl_module --add-module=ngx_aws_auth \ + && make install \ + && cd /tmp \ + && rm -f nginx-$NGINX_VERSION.tar.gz \ + && rm -rf nginx-$NGINX_VERSION \ + && apt-get purge -y curl git \ + && apt-get autoremove -y RUN mkdir -p /data/cache +RUN mkdir -p /etc/nginx/ssl /etc/nginx/sites-enabled \ + && cp /usr/local/nginx/conf/mime.types /etc/nginx/mime.types -CMD [ "/usr/local/nginx/sbin/nginx", "-c", "/nginx.conf" ] \ No newline at end of file +ENV CONFD_VERSION 0.11.0 +ENV CONFD_URL https://github.com/kelseyhightower/confd/releases/download/v$CONFD_VERSION/confd-$CONFD_VERSION-linux-amd64 + +RUN wget --progress dot:mega $CONFD_URL -O /usr/local/bin/confd \ + && chmod 755 /usr/local/bin/confd \ + && mkdir -p /etc/confd/conf.d /etc/confd/templates + +COPY nginx.conf /etc/nginx/nginx.conf +COPY proxy.toml /etc/confd/conf.d/proxy.toml +COPY proxy.tmpl /etc/confd/templates/proxy.tmpl + +EXPOSE 80 +EXPOSE 443 + +CMD confd -onetime -backend env && /usr/local/nginx/sbin/nginx -c /etc/nginx/nginx.conf diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..1c5d379 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 Micah Hausler + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 66e3757..10d4ea2 100644 --- a/README.md +++ b/README.md @@ -1,91 +1,42 @@ +# nginx-s3-proxy ## Motivation -This image was created for use with dogestry. We wanted a caching HTTP proxy between our -servers and S3 so that images were only downloaded once from S3. +This image was created to securely proxy requests to S3. It uses +[confd](http://github.com/kelseyhightower/confd) to update the nginx +configuration and then starts nginx. You'll need to run your container with the +following environment variables: -## Usage - -The image assumes a config file in the container at: `/nginx.conf` so use the `-v` option to -mount one from your host. - - -``` -docker run -p 8000:8000 -v /path/to/nginx.conf:/nginx.conf coopernurse/nginx-s3-proxy -``` - -If you want to store the cache on the host, bind a path to `/data/cache`: - -``` -docker run -p 8000:8000 -v /path/to/nginx.conf:/nginx.conf -v /my/path:/data/cache coopernurse/nginx-s3-proxy ``` - -Feel free to alter the `-p` param if you wish to bind the port differently onto the host. - - -Example nginx.conf file: - +SERVER_NAME +S3_BUCKET +REGION +AWS_ACCESS_KEY_ID +AWS_SECRET_ACCESS_KEY ``` -worker_processes 2; -pid /run/nginx.pid; -daemon off; - -events { - worker_connections 768; -} - -http { - sendfile on; - tcp_nopush on; - tcp_nodelay on; - keepalive_timeout 65; - types_hash_max_size 2048; - server_names_hash_bucket_size 64; - - include /usr/local/nginx/conf/mime.types; - default_type application/octet-stream; - - access_log /usr/local/nginx/logs/access.log; - error_log /usr/local/nginx/logs/error.log; - - gzip on; - gzip_disable "msie6"; - gzip_http_version 1.1; - gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript; - proxy_cache_lock on; - proxy_cache_lock_timeout 60s; - proxy_cache_path /data/cache levels=1:2 keys_zone=s3cache:10m max_size=30g; - server { - listen 8000; - - location / { - proxy_pass https://your-bucket.s3.amazonaws.com; - - aws_access_key your-access-key; - aws_secret_key your-secret-key; - s3_bucket your-bucket; - - proxy_set_header Authorization $s3_auth_token; - proxy_set_header x-amz-date $aws_date; +## Usage - proxy_cache s3cache; - proxy_cache_valid 200 302 24h; - } - } -} +You can run it using a docker-compose like this: + +```yaml +version: '2' +services: + s3proxy: + cpu: 512 + memory: 128m + build: ./ + environment: + SERVER_NAME: "s3proxy.example.com" + S3_BUCKET: "your-conf-data" + REGION: "us-west-2" + AWS_ACCESS_KEY_ID: "" + AWS_SECRET_ACCESS_KEY: "" + ports: + - "8080:80" ``` -Things you want to tweak include: - -* proxy_cache_path - * alter max_size as desired - * if you want the cache stored external to the container, alter the path -* proxy_pass -* aws_access_key -* aws_secret_key -* s3_bucket -* proxy_cache_valid - change 24h to your cache duration as desired. - +## License +MIT diff --git a/nginx.conf b/nginx.conf new file mode 100644 index 0000000..5cc3891 --- /dev/null +++ b/nginx.conf @@ -0,0 +1,83 @@ +# Nginx.conf provided by Docker +user www-data; +worker_processes 2; +pid /var/run/nginx.pid; +daemon off; + +events { + worker_connections 1024; +} + +http { + # Basic Settings + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + server_tokens off; + + types_hash_bucket_size 128; + types_hash_max_size 2048; + + variables_hash_bucket_size 128; + variables_hash_max_size 2048; + + server_names_hash_bucket_size 128; + server_names_hash_max_size 1024; + + proxy_headers_hash_bucket_size 128; + proxy_headers_hash_max_size 1024; + + include /etc/nginx/mime.types; + default_type text/html; + + # Logging Settings + + log_format main '[$time_local] - ' + '$remote_addr - ' + '$status - ' + '"$request" - ' + '$body_bytes_sent - ' + '$cookie_sessionid - ' + '"$http_referer" - ' + '"$http_user_agent" - ' + '$request_time'; + + access_log /dev/stdout main; + error_log /dev/stdout; + + # Gzip Settings + + gzip on; + gzip_min_length 20; + gzip_disable "msie6"; + + gzip_types application/ecmascript; + gzip_types application/javascript; + gzip_types application/json; + gzip_types application/pdf; + gzip_types application/postscript; + gzip_types application/x-javascript; + gzip_types image/svg+xml; + gzip_types image/png; + gzip_types text/css; + gzip_types text/csv; + gzip_types text/javascript; + gzip_types text/plain; + gzip_types text/xml; + + + ## + # Virtual Host Configs + ## + #include /etc/nginx/conf.d/*.conf; + include /etc/nginx/sites-enabled/*; + + # If you use a real certificate + # See https://www.digitalocean.com/community/tutorials/how-to-configure-ocsp-stapling-on-apache-and-nginx + #ssl_stapling on; + #ssl_stapling_verify on; + #ssl_trusted_certificate /etc/nginx/ssl/ca-bundle.crt; + #add_header "Strict-Transport-Security" "max-age=31536000; includeSubDomains; preload"; +} diff --git a/proxy.tmpl b/proxy.tmpl new file mode 100644 index 0000000..7dffbdb --- /dev/null +++ b/proxy.tmpl @@ -0,0 +1,35 @@ +proxy_cache_lock on; +proxy_cache_lock_timeout 60s; +proxy_cache_path /data/cache levels=1:2 keys_zone=s3cache:10m max_size=1g; + +server { + server_name {{ getenv "SERVER_NAME" }}; + + listen 80; + + gzip_min_length 2000; + gzip_proxied any; + + proxy_read_timeout 60s; + client_max_body_size 64M; + + location / { + #if ($scheme = "http") { + # rewrite ^ https://$server_name$request_uri? permanent; + #} + + # https://skuid-conf-data.s3-us-west-2.amazonaws.com/registry/jenkins/docker.tgz + proxy_pass https://{{ getenv "S3_BUCKET" }}.s3-{{ getenv "REGION" }}.amazonaws.com; + + aws_access_key {{ getenv "AWS_ACCESS_KEY_ID" }}; + aws_secret_key {{ getenv "AWS_SECRET_ACCESS_KEY" }}; + s3_bucket {{ getenv "S3_BUCKET" }}; + + proxy_set_header Authorization $s3_auth_token; + proxy_set_header x-amz-date $aws_date; + + proxy_cache s3cache; + proxy_cache_valid 200 302 24h; + } + access_log /dev/stdout main; +} diff --git a/proxy.toml b/proxy.toml new file mode 100644 index 0000000..160bb8c --- /dev/null +++ b/proxy.toml @@ -0,0 +1,12 @@ +[template] +backend = "env" +src = "proxy.tmpl" +dest = "/etc/nginx/sites-enabled/proxy.conf" +owner = "nginx" +mode = "0644" +keys = [ + "SERVER_NAME", + "S3_BUCKET", + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", +] From 5188a9c3f3c2199c47e2ca6e28a4601b45e5dd12 Mon Sep 17 00:00:00 2001 From: Micah Hausler Date: Fri, 17 Jun 2016 15:28:22 -0400 Subject: [PATCH 2/9] Updated compose --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 10d4ea2..1802cc2 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ You can run it using a docker-compose like this: version: '2' services: s3proxy: - cpu: 512 + cpu_shares: 512 memory: 128m build: ./ environment: From 49a07c2404c63f6fe61a4d260616d05bd9343b7a Mon Sep 17 00:00:00 2001 From: Micah Hausler Date: Fri, 17 Jun 2016 15:34:11 -0400 Subject: [PATCH 3/9] updated Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 488b6c7..cefd2ac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ RUN apt-get -y update \ && curl -LO http://nginx.org/download/nginx-$NGINX_VERSION.tar.gz \ && tar zxf nginx-$NGINX_VERSION.tar.gz \ && cd nginx-$NGINX_VERSION \ - && git clone https://github.com/anomalizer/ngx_aws_auth.git \ + && git clone -b AuthV2 https://github.com/anomalizer/ngx_aws_auth.git \ && ./configure --with-http_ssl_module --add-module=ngx_aws_auth \ && make install \ && cd /tmp \ From da3e80f7761759f16a24afc891de3cd2a185a9f7 Mon Sep 17 00:00:00 2001 From: Micah Hausler Date: Fri, 17 Jun 2016 15:43:38 -0400 Subject: [PATCH 4/9] updated compose in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1802cc2..018b3d4 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ version: '2' services: s3proxy: cpu_shares: 512 - memory: 128m + mem_limit: 128m build: ./ environment: SERVER_NAME: "s3proxy.example.com" From 0c4e014f1338dec99fc6be1c24b69640920d0953 Mon Sep 17 00:00:00 2001 From: Micah Hausler Date: Mon, 20 Jun 2016 11:47:05 -0400 Subject: [PATCH 5/9] Configurable cache timeout --- README.md | 2 ++ proxy.tmpl | 3 +-- proxy.toml | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 018b3d4..006fad8 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ configuration and then starts nginx. You'll need to run your container with the following environment variables: ``` +CACHE_TIME SERVER_NAME S3_BUCKET REGION @@ -28,6 +29,7 @@ services: mem_limit: 128m build: ./ environment: + CACHE_TIME: "10m" SERVER_NAME: "s3proxy.example.com" S3_BUCKET: "your-conf-data" REGION: "us-west-2" diff --git a/proxy.tmpl b/proxy.tmpl index 7dffbdb..d1ff28c 100644 --- a/proxy.tmpl +++ b/proxy.tmpl @@ -1,6 +1,6 @@ proxy_cache_lock on; proxy_cache_lock_timeout 60s; -proxy_cache_path /data/cache levels=1:2 keys_zone=s3cache:10m max_size=1g; +proxy_cache_path /data/cache levels=1:2 keys_zone=s3cache:{{ getenv "CACHE_TIME" }} max_size=1g; server { server_name {{ getenv "SERVER_NAME" }}; @@ -18,7 +18,6 @@ server { # rewrite ^ https://$server_name$request_uri? permanent; #} - # https://skuid-conf-data.s3-us-west-2.amazonaws.com/registry/jenkins/docker.tgz proxy_pass https://{{ getenv "S3_BUCKET" }}.s3-{{ getenv "REGION" }}.amazonaws.com; aws_access_key {{ getenv "AWS_ACCESS_KEY_ID" }}; diff --git a/proxy.toml b/proxy.toml index 160bb8c..21aec6f 100644 --- a/proxy.toml +++ b/proxy.toml @@ -5,6 +5,7 @@ dest = "/etc/nginx/sites-enabled/proxy.conf" owner = "nginx" mode = "0644" keys = [ + "CACHE_TIME", "SERVER_NAME", "S3_BUCKET", "AWS_ACCESS_KEY_ID", From 21abd7bc7f56bb9bec01574ebaef5ce7fece6293 Mon Sep 17 00:00:00 2001 From: Micah Hausler Date: Mon, 20 Jun 2016 15:29:09 -0400 Subject: [PATCH 6/9] fixed proxy cache time --- proxy.tmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/proxy.tmpl b/proxy.tmpl index d1ff28c..20b9122 100644 --- a/proxy.tmpl +++ b/proxy.tmpl @@ -1,6 +1,6 @@ proxy_cache_lock on; proxy_cache_lock_timeout 60s; -proxy_cache_path /data/cache levels=1:2 keys_zone=s3cache:{{ getenv "CACHE_TIME" }} max_size=1g; +proxy_cache_path /data/cache levels=1:2 keys_zone=s3cache:10m max_size=1g; server { server_name {{ getenv "SERVER_NAME" }}; @@ -28,7 +28,7 @@ server { proxy_set_header x-amz-date $aws_date; proxy_cache s3cache; - proxy_cache_valid 200 302 24h; + proxy_cache_valid 200 {{ getenv "CACHE_TIME" }}; } access_log /dev/stdout main; } From 17738ded6d55e4f01b08501be8c346a57447bf96 Mon Sep 17 00:00:00 2001 From: Sylvain Zimmer Date: Tue, 2 Aug 2016 01:06:34 -0400 Subject: [PATCH 7/9] Added docker-compose to the repo + CACHE_SIZE variable --- Dockerfile | 2 +- docker-compose.yaml.example | 18 ++++++++++++++++++ proxy.tmpl | 4 ++-- proxy.toml | 1 + 4 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 docker-compose.yaml.example diff --git a/Dockerfile b/Dockerfile index cefd2ac..a38bc8d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM debian:jessie WORKDIR /tmp -ENV NGINX_VERSION 1.11.1 +ENV NGINX_VERSION 1.11.3 RUN apt-get -y update \ && apt-get -y install \ curl \ diff --git a/docker-compose.yaml.example b/docker-compose.yaml.example new file mode 100644 index 0000000..fadf429 --- /dev/null +++ b/docker-compose.yaml.example @@ -0,0 +1,18 @@ +version: '2' +services: + s3proxy: + cpu_shares: 3500 + mem_limit: 3500m + build: ./ + environment: + CACHE_TIME: "10y" + CACHE_SIZE: "900m" + SERVER_NAME: "dumps.commonsearch.org" + S3_BUCKET: "dumps.commonsearch.org" + REGION: "external-1" + AWS_ACCESS_KEY_ID: "" + AWS_SECRET_ACCESS_KEY: "" + ports: + - "80:80" + volumes: + - /data/cache:/data/cache \ No newline at end of file diff --git a/proxy.tmpl b/proxy.tmpl index 20b9122..0ff1d24 100644 --- a/proxy.tmpl +++ b/proxy.tmpl @@ -1,6 +1,6 @@ proxy_cache_lock on; proxy_cache_lock_timeout 60s; -proxy_cache_path /data/cache levels=1:2 keys_zone=s3cache:10m max_size=1g; +proxy_cache_path /data/cache levels=1:2 keys_zone=s3cache:10m max_size={{ getenv "CACHE_SIZE" }}; server { server_name {{ getenv "SERVER_NAME" }}; @@ -11,7 +11,7 @@ server { gzip_proxied any; proxy_read_timeout 60s; - client_max_body_size 64M; + client_max_body_size 1M; location / { #if ($scheme = "http") { diff --git a/proxy.toml b/proxy.toml index 21aec6f..c5310aa 100644 --- a/proxy.toml +++ b/proxy.toml @@ -6,6 +6,7 @@ owner = "nginx" mode = "0644" keys = [ "CACHE_TIME", + "CACHE_SIZE", "SERVER_NAME", "S3_BUCKET", "AWS_ACCESS_KEY_ID", From e809d8e28ad538d93a17fd8711f11fd84d7cb22a Mon Sep 17 00:00:00 2001 From: Sylvain Zimmer Date: Tue, 2 Aug 2016 01:26:47 -0400 Subject: [PATCH 8/9] Tweaks for dumps.commonsearch.org --- docker-compose.yaml.example | 2 +- proxy.tmpl | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docker-compose.yaml.example b/docker-compose.yaml.example index fadf429..15d7b61 100644 --- a/docker-compose.yaml.example +++ b/docker-compose.yaml.example @@ -6,7 +6,7 @@ services: build: ./ environment: CACHE_TIME: "10y" - CACHE_SIZE: "900m" + CACHE_SIZE: "500g" SERVER_NAME: "dumps.commonsearch.org" S3_BUCKET: "dumps.commonsearch.org" REGION: "external-1" diff --git a/proxy.tmpl b/proxy.tmpl index 0ff1d24..ae0619f 100644 --- a/proxy.tmpl +++ b/proxy.tmpl @@ -13,6 +13,10 @@ server { proxy_read_timeout 60s; client_max_body_size 1M; + location = / { + return 302 https://about.commonsearch.org/; + } + location / { #if ($scheme = "http") { # rewrite ^ https://$server_name$request_uri? permanent; From 3d5d372293de5b325474a2a3d347f6d094c31302 Mon Sep 17 00:00:00 2001 From: Sylvain Zimmer Date: Mon, 7 Nov 2016 14:43:14 +0100 Subject: [PATCH 9/9] Fix cache time --- proxy.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxy.tmpl b/proxy.tmpl index ae0619f..c9fd007 100644 --- a/proxy.tmpl +++ b/proxy.tmpl @@ -1,6 +1,6 @@ proxy_cache_lock on; proxy_cache_lock_timeout 60s; -proxy_cache_path /data/cache levels=1:2 keys_zone=s3cache:10m max_size={{ getenv "CACHE_SIZE" }}; +proxy_cache_path /data/cache levels=1:2 keys_zone=s3cache:10m inactive={{ getenv "CACHE_TIME" }} max_size={{ getenv "CACHE_SIZE" }}; server { server_name {{ getenv "SERVER_NAME" }};