https://github.com/osquery/osquery logo
Join Slack
Powered by
Small config dump before I raise an example config...
# fleet
g
Small config dump before I raise an example config. Working ingress-nginx config for k8s
Copy code
apiVersion: <http://networking.k8s.io/v1beta1|networking.k8s.io/v1beta1>
kind: Ingress
metadata:
  name: fleetdm-fleet-ingress
  namespace: default
  annotations:
    <http://kubernetes.io/ingress.class|kubernetes.io/ingress.class>: nginx
    # GRPCS to allow GPRC-SSL for the kolide-launcher used by our instances.
    <http://nginx.ingress.kubernetes.io/backend-protocol|nginx.ingress.kubernetes.io/backend-protocol>: HTTPS
    # NGINX cannot do GPRC-SSL termination passthrough the SSL connection.
    <http://ingress.kubernetes.io/force-ssl-redirect|ingress.kubernetes.io/force-ssl-redirect>: 'true'
    <http://ingress.kubernetes.io/ssl-passthrough|ingress.kubernetes.io/ssl-passthrough>: 'true'
    # <http://nginx.ingress.kubernetes.io/proxy-request-buffering|nginx.ingress.kubernetes.io/proxy-request-buffering>: off
    <http://nginx.ingress.kubernetes.io/server-snippet|nginx.ingress.kubernetes.io/server-snippet>: |
      location /metrics {
        return 403;
      }
      location /debug {
        return 403;
      }
      location /version {
        return 403;
      }

      location /kolide.launcher.QueryTarget/GetTargets {
        port_in_redirect off;

        set $balancer_ewma_score -1;
        set $proxy_upstream_name "default-fleet-loadbalancer-8080";
        set $proxy_host          $proxy_upstream_name;
        set $pass_access_scheme  $scheme;

        set $pass_server_port    $server_port;

        set $best_http_host      $http_host;
        set $pass_port           $pass_server_port;


        grpc_set_header X-Forwarded-For        $remote_addr;

        # In case of errors try the next upstream server before returning an error
        proxy_next_upstream                     error timeout;
        proxy_next_upstream_timeout             0;
        proxy_next_upstream_tries               3;

        grpc_pass <grpcs://upstream_balancer>;

        proxy_redirect                          off;
      }

      location ~ ^/kolide.agent.Api/(RequestEnrollment|RequestConfig|RequestQueries|PublishLogs|PublishResults|CheckHealth)$ {

        port_in_redirect off;

        set $balancer_ewma_score -1;
        set $proxy_upstream_name "default-fleet-loadbalancer-8080";
        set $proxy_host          $proxy_upstream_name;
        set $pass_access_scheme  $scheme;

        set $pass_server_port    $server_port;

        set $best_http_host      $http_host;
        set $pass_port           $pass_server_port;


        grpc_set_header X-Forwarded-For        $remote_addr;

        # In case of errors try the next upstream server before returning an error
        proxy_next_upstream                     error timeout;
        proxy_next_upstream_timeout             0;
        proxy_next_upstream_tries               3;

        grpc_pass <grpcs://upstream_balancer>;

        proxy_redirect                          off;
      }
  labels:
    app: fleet-webserver
....
@hilt you may find this of interest as you were the main person I found asking for it and @defensivedepth thanks for the blog post
d
no problem! 🙂
h
awesome - just what I need! thanks @Gavin + @defensivedepth
g
FYI this started to fall over around 1K hosts I am working on another change this now.
d
@hilt hey hilt, long time no talk 🙂
h
hey @defensivedepth - hope you are going well! I’ve got time this week if you wanted to have a quick chat
👍 1
g
Okay the inverse works of Default GRPCS with a specific route for the required websocket upgrade for the live query Survived a 20K host load-test.
Copy code
---
apiVersion: <http://networking.k8s.io/v1beta1|networking.k8s.io/v1beta1>
kind: Ingress
metadata:
  name: fleet-ingress
  namespace: default
  annotations:
    <http://nginx.ingress.kubernetes.io/proxy-body-size|nginx.ingress.kubernetes.io/proxy-body-size>: "0"
    <http://kubernetes.io/ingress.class|kubernetes.io/ingress.class>: nginx
    # GRPCS to allow GPRC-SSL for the kolide-launcher used by our instances.
    <http://nginx.ingress.kubernetes.io/backend-protocol|nginx.ingress.kubernetes.io/backend-protocol>: GRPCS
    # NGINX cannot do GPRC-SSL termination passthrough the SSL connection.
    <http://ingress.kubernetes.io/force-ssl-redirect|ingress.kubernetes.io/force-ssl-redirect>: "true"
    <http://ingress.kubernetes.io/ssl-passthrough|ingress.kubernetes.io/ssl-passthrough>: "true"
    #<http://nginx.ingress.kubernetes.io/affinity|nginx.ingress.kubernetes.io/affinity>: "cookie"
    #<http://nginx.ingress.kubernetes.io/affinity-mode|nginx.ingress.kubernetes.io/affinity-mode>: "persistent"
    #<http://nginx.ingress.kubernetes.io/proxy-request-buffering|nginx.ingress.kubernetes.io/proxy-request-buffering>: off
    <http://nginx.ingress.kubernetes.io/server-snippet|nginx.ingress.kubernetes.io/server-snippet>: |
      location /metrics {
        return 403;
      }
      location /debug {
        return 403;
      }
      location /version {
        return 403;
      }
      location /api/results {
        set $proxy_alternative_upstream_name "";
        
        # client_max_body_size                    1m;
        
        proxy_set_header Host                   $best_http_host;
         
        # Allow websocket connections
        proxy_set_header                        Upgrade           $http_upgrade;
        
        proxy_set_header                        Connection        $connection_upgrade;
        
        proxy_set_header X-Request-ID           $req_id;
        proxy_set_header X-Real-IP              $remote_addr;
        
        proxy_set_header X-Forwarded-For        $remote_addr;
        
        proxy_set_header X-Forwarded-Host       $best_http_host;
        proxy_set_header X-Forwarded-Port       $pass_port;
        proxy_set_header X-Forwarded-Proto      $pass_access_scheme;
        
        proxy_set_header X-Scheme               $pass_access_scheme;
        
        # Pass the original X-Forwarded-For
        proxy_set_header X-Original-Forwarded-For $http_x_forwarded_for;
        
        proxy_request_buffering                 on;
        proxy_http_version                      1.1;
        
        proxy_pass <https://upstream_balancer>;
        
        proxy_redirect                          off;
      }
  labels:
    app: fleet-webserver
19 Views
Open in Slack
PreviousNext
Powered by