Changed to cache all traffic throught proxy.

README.md

@@ -1,3 +1,6 @@
+Fork of https://github.com/rpardini/docker-registry-proxy.git with minor changes for offline DeckHouse deployment.
+
+
 ![GitHub Workflow Status](https://img.shields.io/github/workflow/status/rpardini/docker-registry-proxy/master-latest?label=%3Alatest%20from%20master)
 ![GitHub tag (latest by date)](https://img.shields.io/github/v/tag/rpardini/docker-registry-proxy?label=last%20tagged%20release)
 ![GitHub Workflow Status](https://img.shields.io/github/workflow/status/rpardini/docker-registry-proxy/tags?label=last%20tagged%20release)

docs/compose/docker-compose.yml

@@ -8,6 +8,9 @@ services:
     environment:
       - CACHE_MAX_SIZE=256g
       - ENABLE_MANIFEST_CACHE=true
+      # Enable/disable internet access, possible variants "on"/"off".
+      # After load all data into cache switch to "off" for offline usage.
+      - PROXY_CACHE_REVALIDATE=on
     volumes:
       # Format: <host-path>:<container-path>; adapt to your needs
       - ./docker_mirror_cache:/docker_mirror_cache # This will be up to CACHE_MAX_SIZE big

entrypoint.sh

@@ -105,43 +105,9 @@ CACHE_MAX_SIZE=${CACHE_MAX_SIZE:-32g}
 echo "proxy_cache_path /docker_mirror_cache levels=1:2 max_size=$CACHE_MAX_SIZE inactive=60d keys_zone=cache:10m use_temp_path=off;" > /etc/nginx/conf.d/cache_max_size.conf
 
 # Manifest caching configuration. We generate config based on the environment vars.
-echo -n "" >/etc/nginx/nginx.manifest.caching.config.conf
+echo -n "" > /etc/nginx/nginx.manifest.caching.config.conf
 
-[[ "a${ENABLE_MANIFEST_CACHE}" == "atrue" ]] && [[ "a${MANIFEST_CACHE_PRIMARY_REGEX}" != "a" ]] && cat <<EOD >>/etc/nginx/nginx.manifest.caching.config.conf
+echo -n "proxy_cache_revalidate ${PROXY_CACHE_REVALIDATE};" > /etc/nginx/nginx.cache.revalidate.conf
-    # First tier caching of manifests; configure via MANIFEST_CACHE_PRIMARY_REGEX and MANIFEST_CACHE_PRIMARY_TIME
-    location ~ ^/v2/(.*)/manifests/${MANIFEST_CACHE_PRIMARY_REGEX} {
-        set \$docker_proxy_request_type "manifest-primary";
-        proxy_cache_valid ${MANIFEST_CACHE_PRIMARY_TIME};
-        include "/etc/nginx/nginx.manifest.stale.conf";
-    }
-EOD
-
-[[ "a${ENABLE_MANIFEST_CACHE}" == "atrue" ]] && [[ "a${MANIFEST_CACHE_SECONDARY_REGEX}" != "a" ]] && cat <<EOD >>/etc/nginx/nginx.manifest.caching.config.conf
-    # Secondary tier caching of manifests; configure via MANIFEST_CACHE_SECONDARY_REGEX and MANIFEST_CACHE_SECONDARY_TIME
-    location ~ ^/v2/(.*)/manifests/${MANIFEST_CACHE_SECONDARY_REGEX} {
-        set \$docker_proxy_request_type "manifest-secondary";
-        proxy_cache_valid ${MANIFEST_CACHE_SECONDARY_TIME};
-        include "/etc/nginx/nginx.manifest.stale.conf";
-    }
-EOD
-
-[[ "a${ENABLE_MANIFEST_CACHE}" == "atrue" ]] && cat <<EOD >>/etc/nginx/nginx.manifest.caching.config.conf
-    # Default tier caching for manifests. Caches for ${MANIFEST_CACHE_DEFAULT_TIME} (from MANIFEST_CACHE_DEFAULT_TIME)
-    location ~ ^/v2/(.*)/manifests/ {
-        set \$docker_proxy_request_type "manifest-default";
-        proxy_cache_valid ${MANIFEST_CACHE_DEFAULT_TIME};
-        include "/etc/nginx/nginx.manifest.stale.conf";
-    }
-EOD
-
-[[ "a${ENABLE_MANIFEST_CACHE}" != "atrue" ]] && cat <<EOD >>/etc/nginx/nginx.manifest.caching.config.conf
-    # Manifest caching is disabled. Enable it with ENABLE_MANIFEST_CACHE=true
-    location ~ ^/v2/(.*)/manifests/ {
-        set \$docker_proxy_request_type "manifest-default-disabled";
-        proxy_cache_valid 0s;
-        include "/etc/nginx/nginx.manifest.stale.conf";
-    }
-EOD
 
 echo -e "\nManifest caching config: ---\n"
 cat /etc/nginx/nginx.manifest.caching.config.conf
@@ -152,8 +118,8 @@ if [[ "a${ALLOW_PUSH}" == "atrue" ]]; then
     # allow to upload big layers
     client_max_body_size 0;
 
-    # only cache GET requests
+    # only cache GET HEAD requests
-    proxy_cache_methods GET;
+    proxy_cache_methods GET HEAD;
 EOF
 else
     cat << 'EOF' > /etc/nginx/conf.d/allowed.methods.conf
@@ -288,4 +254,4 @@ echo "Testing nginx config..."
 ${NGINX_BIN} -t
 
 echo "Starting nginx! Have a nice day."
-${NGINX_BIN} -g "daemon off;"
+${NGINX_BIN} -g "daemon off;"

nginx.conf

@@ -236,13 +236,14 @@ echo "Docker configured with HTTPS_PROXY=$scheme://$http_host/"
         proxy_cache_lock on;
         proxy_cache_lock_timeout 880s;
 
-        # Cache all 200, 206 for 60 days.
+        # Cache all 200, 206 for 720d days.
-        proxy_cache_valid 200 206 60d;
+        proxy_cache_valid 200 206 720d;
 
         # Some extra settings to maximize cache hits and efficiency
         proxy_force_ranges on;
         proxy_ignore_client_abort on;
-        proxy_cache_revalidate on;
+
+        include "/etc/nginx/nginx.cache.revalidate.conf";
 
         # Hide/ignore headers from caching. S3 especially likes to send Expires headers in the past in some situations.
         proxy_hide_header      Set-Cookie;
@@ -264,31 +265,16 @@ echo "Docker configured with HTTPS_PROXY=$scheme://$http_host/"
         }
 
         # For blob requests by digest, do cache, and treat redirects.
-        location ~ ^/v2/(.*)/blobs/sha256:(.*) {
+        location / {
-            set $docker_proxy_request_type "blob-by-digest";
+            set $docker_proxy_request_type "proxy";
-            include "/etc/nginx/nginx.manifest.common.conf";
-        }
-
-        # For manifest requests by digest, do cache, and treat redirects.
-        # These are some of the requests that DockerHub will throttle.
-        location ~ ^/v2/(.*)/manifests/sha256:(.*) {
-            set $docker_proxy_request_type "manifest-by-digest";
             include "/etc/nginx/nginx.manifest.common.conf";
+            proxy_cache_use_stale error timeout http_500 http_502 http_504 http_429;
         }
 
         # Config for manifest URL caching is generated by the entrypoint based on ENVs.
         # Go check it out, entrypoint.sh
         include "/etc/nginx/nginx.manifest.caching.config.conf";
 
-
-        # Cache blobs requests that are not by digest
-        # Since these are mutable, we invalidate them immediately and keep them only in case the backend is down
-        location ~ ^/v2/(.*)/blobs/ {
-            set $docker_proxy_request_type "blob-mutable";
-            proxy_cache_valid 0s;
-            include "/etc/nginx/nginx.manifest.stale.conf";
-        }
-
         location @handle_redirects {
             #store the current state of the world so we can reuse it in a minute
             # We need to capture these values now, because as soon as we invoke
@@ -309,11 +295,5 @@ echo "Docker configured with HTTPS_PROXY=$scheme://$http_host/"
             # so that future clients don't need to follow the redirect too
             proxy_cache_key $original_uri;
         }
-
-        # by default, dont cache anything.
-        location / {
-            proxy_pass https://$targetHost;
-            proxy_cache off;
-        }
     }
 }

nginx.manifest.common.conf

@@ -3,6 +3,6 @@
     add_header X-Docker-Registry-Proxy-Cache-Type "$docker_proxy_request_type";
     proxy_pass https://$targetHost;
     proxy_cache cache;
-    proxy_cache_key   $uri;
+    proxy_cache_key $host$uri;
     proxy_intercept_errors on;
     error_page 301 302 307 = @handle_redirects;

nginx.manifest.stale.conf

@@ -1,3 +1,3 @@
     # Just like the common block, but adds proxy_cache_use_stale
     include "/etc/nginx/nginx.manifest.common.conf";
-    proxy_cache_use_stale  error timeout http_500 http_502 http_504 http_429;
+    proxy_cache_use_stale error timeout http_500 http_502 http_504 http_429;