Channels
android_tests
apple-silicon
arm-architecture
auditing-warroom
aws
carving
code-review
community-feeds
core
darkbytes
doorman
ebpf
eclecticiq-polylogyx-extension
extensions
file-carving
fim
fleet
fleet-dev
fleetosquery
foundation
fuzzing
general
golang
goquery
infrastructure
jobs
kolide
linen-dev
linux
macos
officehours
osctrl
plugins
process-auditing
querycon
queryhub
random
selfgroup
sql
tls
uptycs
vendor-feeds
website
windows
zeek
zentral
zercurity
Title
g
Gavin
01/25/2021, 9:38 PMSmall config dump before I raise an example config.
Working ingress-nginx config for k8s
apiVersion: <http://networking.k8s.io/v1beta1|networking.k8s.io/v1beta1>
kind: Ingress
metadata:
name: fleetdm-fleet-ingress
namespace: default
annotations:
<http://kubernetes.io/ingress.class|kubernetes.io/ingress.class>: nginx
# GRPCS to allow GPRC-SSL for the kolide-launcher used by our instances.
<http://nginx.ingress.kubernetes.io/backend-protocol|nginx.ingress.kubernetes.io/backend-protocol>: HTTPS
# NGINX cannot do GPRC-SSL termination passthrough the SSL connection.
<http://ingress.kubernetes.io/force-ssl-redirect|ingress.kubernetes.io/force-ssl-redirect>: 'true'
<http://ingress.kubernetes.io/ssl-passthrough|ingress.kubernetes.io/ssl-passthrough>: 'true'
# <http://nginx.ingress.kubernetes.io/proxy-request-buffering|nginx.ingress.kubernetes.io/proxy-request-buffering>: off
<http://nginx.ingress.kubernetes.io/server-snippet|nginx.ingress.kubernetes.io/server-snippet>: |
location /metrics {
return 403;
}
location /debug {
return 403;
}
location /version {
return 403;
}
location /kolide.launcher.QueryTarget/GetTargets {
port_in_redirect off;
set $balancer_ewma_score -1;
set $proxy_upstream_name "default-fleet-loadbalancer-8080";
set $proxy_host $proxy_upstream_name;
set $pass_access_scheme $scheme;
set $pass_server_port $server_port;
set $best_http_host $http_host;
set $pass_port $pass_server_port;
grpc_set_header X-Forwarded-For $remote_addr;
# In case of errors try the next upstream server before returning an error
proxy_next_upstream error timeout;
proxy_next_upstream_timeout 0;
proxy_next_upstream_tries 3;
grpc_pass <grpcs://upstream_balancer>;
proxy_redirect off;
}
location ~ ^/kolide.agent.Api/(RequestEnrollment|RequestConfig|RequestQueries|PublishLogs|PublishResults|CheckHealth)$ {
port_in_redirect off;
set $balancer_ewma_score -1;
set $proxy_upstream_name "default-fleet-loadbalancer-8080";
set $proxy_host $proxy_upstream_name;
set $pass_access_scheme $scheme;
set $pass_server_port $server_port;
set $best_http_host $http_host;
set $pass_port $pass_server_port;
grpc_set_header X-Forwarded-For $remote_addr;
# In case of errors try the next upstream server before returning an error
proxy_next_upstream error timeout;
proxy_next_upstream_timeout 0;
proxy_next_upstream_tries 3;
grpc_pass <grpcs://upstream_balancer>;
proxy_redirect off;
}
labels:
app: fleet-webserver
....@hilt you may find this of interest as you were the main person I found asking for it and @defensivedepth thanks for the blog post
d
defensivedepth
01/25/2021, 9:51 PMno problem! 🙂
h
hilt
01/26/2021, 12:02 AMawesome - just what I need! thanks @Gavin + @defensivedepth
g
Gavin
01/26/2021, 12:03 AMFYI this started to fall over around 1K hosts I am working on another change this now.
d
defensivedepth
01/26/2021, 12:04 AM@hilt hey hilt, long time no talk 🙂
h
hilt
01/26/2021, 12:08 AMhey @defensivedepth - hope you are going well! I’ve got time this week if you wanted to have a quick chat
👍 1
g
Gavin
01/26/2021, 12:36 AMOkay the inverse works of Default GRPCS with a specific route for the required websocket upgrade for the live query
Survived a 20K host load-test.
---
apiVersion: <http://networking.k8s.io/v1beta1|networking.k8s.io/v1beta1>
kind: Ingress
metadata:
name: fleet-ingress
namespace: default
annotations:
<http://nginx.ingress.kubernetes.io/proxy-body-size|nginx.ingress.kubernetes.io/proxy-body-size>: "0"
<http://kubernetes.io/ingress.class|kubernetes.io/ingress.class>: nginx
# GRPCS to allow GPRC-SSL for the kolide-launcher used by our instances.
<http://nginx.ingress.kubernetes.io/backend-protocol|nginx.ingress.kubernetes.io/backend-protocol>: GRPCS
# NGINX cannot do GPRC-SSL termination passthrough the SSL connection.
<http://ingress.kubernetes.io/force-ssl-redirect|ingress.kubernetes.io/force-ssl-redirect>: "true"
<http://ingress.kubernetes.io/ssl-passthrough|ingress.kubernetes.io/ssl-passthrough>: "true"
#<http://nginx.ingress.kubernetes.io/affinity|nginx.ingress.kubernetes.io/affinity>: "cookie"
#<http://nginx.ingress.kubernetes.io/affinity-mode|nginx.ingress.kubernetes.io/affinity-mode>: "persistent"
#<http://nginx.ingress.kubernetes.io/proxy-request-buffering|nginx.ingress.kubernetes.io/proxy-request-buffering>: off
<http://nginx.ingress.kubernetes.io/server-snippet|nginx.ingress.kubernetes.io/server-snippet>: |
location /metrics {
return 403;
}
location /debug {
return 403;
}
location /version {
return 403;
}
location /api/results {
set $proxy_alternative_upstream_name "";
# client_max_body_size 1m;
proxy_set_header Host $best_http_host;
# Allow websocket connections
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
proxy_set_header X-Request-ID $req_id;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $remote_addr;
proxy_set_header X-Forwarded-Host $best_http_host;
proxy_set_header X-Forwarded-Port $pass_port;
proxy_set_header X-Forwarded-Proto $pass_access_scheme;
proxy_set_header X-Scheme $pass_access_scheme;
# Pass the original X-Forwarded-For
proxy_set_header X-Original-Forwarded-For $http_x_forwarded_for;
proxy_request_buffering on;
proxy_http_version 1.1;
proxy_pass <https://upstream_balancer>;
proxy_redirect off;
}
labels:
app: fleet-webserver