User guide

kind: ConfigMap
apiVersion: v1
metadata:
  name: nginx-configuration
data:
  access-log-path: "/var/log/nginx/access.log"
  error-log-path: "/var/log/nginx/error.log"

"load_module,lua_package,_by_lua,location,root,proxy_pass,serviceaccount,{,},',\""

$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_length $request_time [$proxy_upstream_name] [$proxy_alternative_upstream_name] $upstream_addr $upstream_response_length $upstream_response_time $upstream_status $req_id

[$remote_addr] [$time_local] $protocol $status $bytes_sent $bytes_received $session_time

application/atom+xml application/javascript application/x-javascript application/json application/rss+xml application/vnd.ms-fontobject application/x-font-ttf application/x-web-app-manifest+json application/xhtml+xml application/xml font/opentype image/svg+xml image/x-icon text/css text/plain text/x-component

/etc/nginx/template/nginx.tmpl

apiVersion: v1
kind: ConfigMap
metadata:
  name: nginx-template
  namespace: kube-system
data:
  nginx.tmpl: |
      {{ $all := . }}
      {{ $servers := .Servers }}
      {{ $cfg := .Cfg }}
      {{ $IsIPV6Enabled := .IsIPV6Enabled }}
      {{ $healthzURI := .HealthzURI }}
      {{ $backends := .Backends }}
      {{ $proxyHeaders := .ProxySetHeaders }}
      {{ $addHeaders := .AddHeaders }}

      # Configuration checksum: {{ $all.Cfg.Checksum }}
      ......

        volumeMounts:
        - name: nginx-template
          mountPath: /etc/nginx/template
          readOnly: true
......
      volumes:
      - name: nginx-template
        configMap:
          name: nginx-template
......

kind: ConfigMap
apiVersion: v1
metadata:
  name: tcp-services
data:
  "1053": "kube-system/kube-dns:53"

stream {
      ......
      server {
            preread_by_lua_block {
                  ngx.var.proxy_upstream_name="tcp-kube-system-kube-dns-53";
            }

            listen                  1053;

            proxy_timeout           600s;
            proxy_next_upstream     on;
            proxy_next_upstream_timeout 600s;
            proxy_next_upstream_tries   3;

            proxy_pass              upstream_balancer;
      }
      ......
}

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  annotations:
    nginx.ingress.kubernetes.io/app-root: "/test"

{{ if not (empty $location.Rewrite.AppRoot) }}
if ($uri = /) {
      return 302 $scheme://$http_host{{ $location.Rewrite.AppRoot }};
}
{{ end }}

server {
      if ($uri = /) {
            return 302 $scheme://$http_host/test;
      }
}

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  annotations:
    nginx.ingress.kubernetes.io/affinity: "cookie"
    nginx.ingress.kubernetes.io/session-cookie-name: "HELLO"

< HTTP/1.1 200 OK
< Date: Mon, 12 Sep 2022 14:05:36 GMT
< Content-Type: text/html
< Content-Length: 4
< Connection: keep-alive
< Set-Cookie: HELLO=1662991537.226.193.550764|a6b1ec86db225f232b62b29085af45ff; Path=/; HttpOnly
< Last-Modified: Mon, 11 Dec 2017 03:18:39 GMT

htpasswd -c http-auth.txt user1

# 用户: user1
# 密码: kubernetes
user1:$apr1$ue3Fu891$xnyqlJlwcM4iWtJrAN8ZW/

cat http-auth.txt

apiVersion: v1
kind: Secret
metadata:
  name: basic-auth
stringData:
  auth: dXNlcjE6JGFwcjEkdWUzRnU4OTEkeG55cWxKbHdjTTRpV3RKckFOOFpXLwo=
type: Opaque

apiVersion: v1
kind: Secret
metadata:
  name: basic-auth
stringData:
  user1: $apr1$ue3Fu891$xnyqlJlwcM4iWtJrAN8ZW/
type: Opaque

log-format-escape-json: "true"
log-format-upstream: '{"msec": "$msec", "remote_addr": "$remote_addr", "request_length": $request_length, "request_method": "$request_method", "host": "$host", "request_uri": "$request_uri", "server_name": "$server_name", "server_addr": "$server_addr", "server_port": $server_port, "uri": "$uri", "args": "$args", "server_protocol": "$server_protocol", "bytes_sent": $bytes_sent, "body_bytes_sent": $body_bytes_sent, "status": $status, "request_time": $request_time, "http_referer": "$http_referer", "http_user_agent": "$http_user_agent", "http_x_tr_user_id": "$http_x_tr_user_id", "http_x_tr_pv_id": "$http_x_tr_pv_id", "http_x_tr_request_id": "$req_id", "http_x_tr_rpc_id": "$http_x_tr_rpc_id", "proxy_upstream_name": "$proxy_upstream_name", "upstream_addr": "$upstream_addr", "upstream_response_length": "$upstream_response_length", "upstream_response_time": "$upstream_response_time", "upstream_status": "$upstream_status"}'

kind: ConfigMap
apiVersion: v1
metadata:
  name: nginx-configuration
data:
  allow-snippet-annotations: "true"

nginx.ingress.kubernetes.io/configuration-snippet
nginx.ingress.kubernetes.io/server-snippet
nginx.ingress.kubernetes.io/stream-snippet
nginx.ingress.kubernetes.io/modsecurity-snippet

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  annotations:
    nginx.ingress.kubernetes.io/configuration-snippet: |
      more_set_headers "x-tr-request-id: $req_id";

http {
    ## start server {{ $server.Hostname }}
    server {
        ......
        location {{ $path }} {

            {{/* Add any additional configuration defined */}}
            {{ $location.ConfigurationSnippet }}

        }
        ......
    }
    ## end server {{ $server.Hostname }}
}

http {
    server {
        ......
        location / {

            more_set_headers "Request-Id: $req_id";

        }
        ......
    }
}

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  annotations:
    nginx.ingress.kubernetes.io/server-snippet: |
        set $agentflag 0;

        if ($http_user_agent ~* "(Mobile)" ){
          set $agentflag 1;
        }

        if ( $agentflag = 1 ) {
          return 301 https://m.a-demo-web.site;
        }

http {
    ## start server {{ $server.Hostname }}
    server {
        ......

        {{ if not (empty $cfg.ServerSnippet) }}
        # Custom code snippet configured in the configuration configmap
        {{ $cfg.ServerSnippet }}
        {{ end }}

        ......
    }
    ## end server {{ $server.Hostname }}
}

http {
    server {

        # Custom code snippet configured for host api.a-demo-web.site
        set $agentflag 0;

        if ($http_user_agent ~* "(Mobile)" ){
                set $agentflag 1;
        }

        if ( $agentflag = 1 ) {
                return 301 https://m.a-demo-web.site;
        }

    }
}

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  annotations:
    nginx.ingress.kubernetes.io/stream-snippet: |
      server {
        listen 8000;
        proxy_pass 127.0.0.1:80;
      }

stream {
    # TCP services
    {{ range $tcpServer := .TCPBackends }}
    server {
        ....
    }
    {{ end }}

    # UDP services
    {{ range $udpServer := .UDPBackends }}
    server {
        ....
    }
    {{ end }}

    # Stream Snippets
    {{ range $snippet := .StreamSnippets }}
    {{ $snippet }}
    {{ end }}
}

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  annotations:
    nginx.ingress.kubernetes.io/auth-url: http://a-demo-web.site/external-auth
    nginx.ingress.kubernetes.io/auth-snippet: |
      proxy_set_header Foo-Header 42;

http {
    ## start server {{ $server.Hostname }}
    server {

        {{ if $authPath }}
        location = {{ $authPath }} {
            internal;

            {{ if not (empty $externalAuth.AuthSnippet) }}
            {{ $externalAuth.AuthSnippet }}
            {{ end }}

        }

    }
    ## end server {{ $server.Hostname }}
}

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  annotations:
    nginx.ingress.kubernetes.io/enable-modsecurity: "true"
    nginx.ingress.kubernetes.io/modsecurity-snippet: |
      SecRuleEngine On
      SecDebugLog /tmp/modsec_debug.log

http {
    ## start server {{ $server.Hostname }}
    server {
        ......
        location {{ $path }} {

            {{ buildModSecurityForLocation $all.Cfg $location }}

        }
        ......
    }
    ## end server {{ $server.Hostname }}
}

# 各各站点的配置生成
# internal/ingress/controller/template/template.go

func buildModSecurityForLocation(cfg config.Configuration, location *ingress.Location) string {
        isMSEnabledInLoc := location.ModSecurity.Enable
        isMSEnableSetInLoc := location.ModSecurity.EnableSet
        isMSEnabled := cfg.EnableModsecurity

        ......

        if location.ModSecurity.Snippet != "" {
                buffer.WriteString(fmt.Sprintf(`modsecurity_rules '
%v
';
`, location.ModSecurity.Snippet))
        }

        return buffer.String()
}

http {
    server {
        ......
        set $proxy_alternative_upstream_name "";

        modsecurity on;
        modsecurity_rules '
        SecRuleEngine On
        SecDebugLog /tmp/modsec_debug.log

        ';

        client_max_body_size                    20m;
        ......
    }
}

main-snippet
http-snippet
server-snippet
stream-snippet
location-snippet

......
daemon off;

# main-snippet
{{ if not (empty $cfg.MainSnippet) }}
{{ $cfg.MainSnippet }}
{{ end }}

events {
}

http {

    # http-snippet
    {{ if not (empty $cfg.HTTPSnippet) }}
    # Custom code snippet configured in the configuration configmap
    {{ $cfg.HTTPSnippet }}
    {{ end }}

    ## start server {{ $server.Hostname }}
    server {
        ......
        # 该模版内同样包含一个"server snippet"，由客户端通过注解提交
        # 以下属于 {{ template "SERVER" serverConfig $all $server }} 内容

        {{ if not (empty $server.ServerSnippet) }}
        # Custom code snippet configured for host {{ $server.Hostname }}
        {{ $server.ServerSnippet }}
        {{ end }}

        location {{ $path }} {

            # location-snippet
            {{ if not (empty $all.Cfg.LocationSnippet) }}
            # Custom code snippet configured in the configuration configmap
            {{ $all.Cfg.LocationSnippet }}
            {{ end }}
        }        
        # 以上属于 {{ template "SERVER" serverConfig $all $server }} 内容

        # server-snippet
        {{ if not (empty $cfg.ServerSnippet) }}
        # Custom code snippet configured in the configuration configmap
        {{ $cfg.ServerSnippet }}
        {{ end }}
    
        {{ template "CUSTOM_ERRORS" (buildCustomErrorDeps "upstream-default-backend" $cfg.CustomHTTPErrors $all.EnableMetrics $cfg.EnableModsecurity) }}
    }
    ## end server {{ $server.Hostname }}
}

stream {
    ......

    # stream-snippet
    # 客户端通过ingress提交注解也在次
    # Stream Snippets
    {{ range $snippet := .StreamSnippets }}
    {{ $snippet }}
    {{ end }}
}
......

# backend for when default-backend-service is not configured or it does not have endpoints
server {
    listen 8181 default_server reuseport backlog=4096;

    set $proxy_upstream_name "internal";

    access_log off;

    location / {
        return 404;
    }
}

if conf.EnableProfiling {
    go metrics.RegisterProfiler("127.0.0.1", nginx.ProfilerPort)
}

# default server, used for NGINX healthcheck and access to nginx stats
server {
    listen 127.0.0.1:{{ .StatusPort }};
    set $proxy_upstream_name "internal";

    keepalive_timeout 0;
    gzip off;

    access_log off;

    location {{ $healthzURI }} {
        return 200;
    }

    location /is-dynamic-lb-initialized {
        ......
    }

    location {{ .StatusPath }} {
        stub_status on;
    }

    location /configuration {
        ......
    }

    location / {
        content_by_lua_block {
            ngx.exit(ngx.HTTP_NOT_FOUND)
        }
    }
}

streams := make([]ingress.Backend, 0)

for _, ep := range TCPEndpoints {
    var service *apiv1.Service
    if ep.Service != nil {
        service = &apiv1.Service{Spec: ep.Service.Spec}
    }

    key := fmt.Sprintf("tcp-%v-%v-%v", ep.Backend.Namespace, ep.Backend.Name, ep.Backend.Port.String())
    streams = append(streams, ingress.Backend{
        Name:      key,
        Endpoints: ep.Endpoints,
        Port:      intstr.FromInt(ep.Port),
        Service:   service,
    })
}

for _, ep := range UDPEndpoints {
    var service *apiv1.Service
    if ep.Service != nil {
        service = &apiv1.Service{Spec: ep.Service.Spec}
    }

    key := fmt.Sprintf("udp-%v-%v-%v", ep.Backend.Namespace, ep.Backend.Name, ep.Backend.Port.String())
    streams = append(streams, ingress.Backend{
        Name:      key,
        Endpoints: ep.Endpoints,
        Port:      intstr.FromInt(ep.Port),
        Service:   service,
    })
}

buf, err := json.Marshal(streams)

hostPort := net.JoinHostPort("127.0.0.1", fmt.Sprintf("%v", nginx.StreamPort))
conn, err := net.Dial("tcp", hostPort)

_, err = conn.Write(buf)

kube-system/ingress-nginx-defaultbackend

gcr.io/google-containers/defaultbackend:1.4

https://github.com/kubernetes/ingress-gce/tree/master/cmd/404-server

args:
- /nginx-ingress-controller
- --default-backend-service=$(POD_NAMESPACE)/ingress-nginx-defaultbackend

<html>
<head><title>404 Not Found</title></head>
<body>
<center><h1>404 Not Found</h1></center>
<hr><center>nginx</center>
</body>
</html>

default backend - 404

<html>
<head><title>504 Gateway Time-out</title></head>
<body>
<center><h1>504 Gateway Time-out</h1></center>
<hr><center>nginx</center>
</body>
</html>

/etc/nginx $ sysctl -a | grep kernel.core_uses_pid
kernel.core_uses_pid = 0
/etc/nginx $
/etc/nginx $ sysctl -a | grep net.core.somaxconn
net.core.somaxconn = 4096
/etc/nginx $
/etc/nginx $ sysctl -a | grep net.ipv4.ip_local_port_range
net.ipv4.ip_local_port_range = 32768	60999
/etc/nginx $ ^C

apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  annotations:
    nginx.ingress.kubernetes.io/rewrite-target: "/$1"
  name: commons-api
spec:
  ingressClassName: nginx
  rules:
  - host: api.a-demo-web.site
    http:
      paths:
      - path: /commons/cmdb/v1/(.*)
        backend:
          serviceName: commons-cmdb-v1
          servicePort: 10080
        pathType: Prefix

curl -v -H "X-Forwarded-For: 1.1.1.1, 2.2.2.2" https://www.baidu.com

forwarded-for-header: "X-Forwarded-For"
use-forwarded-headers: "true"
compute-full-forwarded-for: "true"
proxy-real-ip-cidr: ""

http {

......

    {{ if and $cfg.UseForwardedHeaders $cfg.ComputeFullForwardedFor }}
    # We can't use $proxy_add_x_forwarded_for because the realip module
    # replaces the remote_addr too soon
    map $http_x_forwarded_for $full_x_forwarded_for {
        {{ if $all.Cfg.UseProxyProtocol }}
        default          "$http_x_forwarded_for, $proxy_protocol_addr";
        ''               "$proxy_protocol_addr";
        {{ else }}
        default          "$http_x_forwarded_for, $realip_remote_addr";
        ''               "$realip_remote_addr";
        {{ end}}
    }

......

}

{{/* definition of server-template to avoid repetitions with server-alias */}}
{{ define "SERVER" }}

......

        {{ if $authPath }}
        location = {{ $authPath }} {

            {{ if and $all.Cfg.UseForwardedHeaders $all.Cfg.ComputeFullForwardedFor }}
            proxy_set_header            X-Forwarded-For        $full_x_forwarded_for;
            {{ else }}
            proxy_set_header            X-Forwarded-For        $remote_addr;
            {{ end }}

        }
        {{ end }}

        location {{ $path }} {

			# 如果同时设置了 "use-forwarded-headers" 与 "compute-full-forwarded-for" 则不进行 xff 的替换。
            {{ if and $all.Cfg.UseForwardedHeaders $all.Cfg.ComputeFullForwardedFor }}
            {{ $proxySetHeader }} X-Forwarded-For        $full_x_forwarded_for;
            {{ else }}
            {{ $proxySetHeader }} X-Forwarded-For        $remote_addr;
            {{ end }}


        }

......

{{ end }}