We have a printing application running behind an nginx, and since moving to a new proxy server with a newer nginx, customers are intermittently reporting
System.Net.WebSockets.WebSocketException (0x80004005): The remote party closed the WebSocket connection without completing the close handshake.
at Domain.Printing.Service.Library.SignalRClient.Execute()
and
System.Net.WebSockets.WebSocketException (0x80004005): Unable to connect to the remote server ---> System.Net.WebException: The remote server returned an error: (503) Server Unavailable.
which I believe are connected to these messages in the application and nginx logfiles:
Application log:
2024-06-17 08:58:08 10.24.11.16 POST /printing/SignalR/signalr/abort clientProtocol=2.1&transport=webSockets&connectionData=[%7B%22Name%22:%22IntegratedPrintingHub%22%7D]&connectionToken=mLUYvm 80 - 10.24.11.15 SignalR.Client.NetStandard/2.4.1.0+(Microsoft+Windows+NT+6.2.9200.0) - 200 0 0 0
Nginx error log:
2024/06/17 08:58:08 [error] 892#892: *3890934 recv() failed (104: Unknown error) while proxying upgraded connection, client: 66.66.66.66, server: printing.domain.info, request: "GET /printing/SignalR/signalr/connect?clientProtocol=2.1&transport=webSockets&connectionData=[%7B%22Name%22:%22IntegratedPrintingHub%22%7D]&connectionToken=mLUYvm HTTP/1.1", upstream: "http://10.24.11.16:80/printing/SignalR/signalr/connect?clientProtocol=2.1&transport=webSockets&connectionData=[%7B%22Name%22:%22IntegratedPrintingHub%22%7D]&connectionToken=mLUYvm", host: "printing.domain.info"
nginx -T:
nginx: [warn] duplicate value "TLSv1" in /etc/nginx/sites-enabled/domain.info:38
nginx: [warn] duplicate value "TLSv1.1" in /etc/nginx/sites-enabled/domain.info:38
nginx: [warn] duplicate value "TLSv1.2" in /etc/nginx/sites-enabled/domain.info:38
nginx: [warn] duplicate value "TLSv1.3" in /etc/nginx/sites-enabled/domain.info:38
nginx: the configuration file /etc/nginx/nginx.conf syntax is ok
nginx: configuration file /etc/nginx/nginx.conf test is successful
# configuration file /etc/nginx/nginx.conf:
user www-data;
worker_processes auto;
pid /run/nginx.pid;
include /etc/nginx/modules-enabled/*.conf;
events {
worker_connections 7680;
# multi_accept on;
}
http {
##
# Basic Settings
##
sendfile on;
tcp_nopush on;
types_hash_max_size 2048;
# server_tokens off;
# server_names_hash_bucket_size 64;
# server_name_in_redirect off;
include /etc/nginx/mime.types;
default_type application/octet-stream;
##
# SSL Settings
##
ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3; # Dropping SSLv3, ref: POODLE
ssl_prefer_server_ciphers on;
##
# Logging Settings
##
access_log /var/log/nginx/access.log;
error_log /var/log/nginx/error.log;
##
# Gzip Settings
##
gzip on;
# gzip_vary on;
# gzip_proxied any;
# gzip_comp_level 6;
# gzip_buffers 16 8k;
# gzip_http_version 1.1;
# gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;
##
# Virtual Host Configs
##
include /etc/nginx/conf.d/*.conf;
include /etc/nginx/sites-enabled/*;
}
# configuration file /etc/nginx/modules-enabled/50-mod-http-geoip2.conf:
load_module modules/ngx_http_geoip2_module.so;
# configuration file /etc/nginx/modules-enabled/50-mod-http-image-filter.conf:
load_module modules/ngx_http_image_filter_module.so;
# configuration file /etc/nginx/modules-enabled/50-mod-http-xslt-filter.conf:
load_module modules/ngx_http_xslt_filter_module.so;
# configuration file /etc/nginx/modules-enabled/50-mod-mail.conf:
load_module modules/ngx_mail_module.so;
# configuration file /etc/nginx/modules-enabled/50-mod-stream.conf:
load_module modules/ngx_stream_module.so;
# configuration file /etc/nginx/modules-enabled/70-mod-stream-geoip2.conf:
load_module modules/ngx_stream_geoip2_module.so;
# configuration file /etc/nginx/sites-enabled/domain.info:
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;
}
log_format lglog '[$time_local] $remote_addr - $remote_user - $server_name $host to: $upstream_addr: $request $status upstream_response_time $upstream_response_time msec $msec request_time $request_time';
upstream printingbackend2 {
server ltweb05.domain.local max_fails=3 fail_timeout=10s;
}
ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3;
ssl_ciphers "EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH:@SECLEVEL=0";
ssl_certificate /etc/nginx/ssl/star.domain.info.crt;
ssl_certificate_key /etc/nginx/ssl/star.domain.info.key;
#ssl_dhparam /etc/nginx/ssl/dhparams.pem;
# cache-buffers
proxy_buffering on;
proxy_cache_valid any 10m;
proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=lg-cache:8m max_size=1000m inactive=600m;
proxy_temp_path /var/cache/nginx/tmp;
proxy_buffer_size 4k;
proxy_buffers 100 8k;
client_max_body_size 100M;
server {
listen 80;
listen 443 ssl;
server_name printing.domain.info;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Proto $scheme;
proxy_ssl_server_name on;
add_header via "domain proxy ($proxy_host, $upstream_addr)";
client_header_timeout 45m;
access_log /var/log/nginx/printing.domain.info-access.log;
error_log /var/log/nginx/printing.domain.info-error.log;
location / {
proxy_pass http://printingbackend2;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
#http://nginx.org/en/docs/http/websocket.html
#proxy_set_header Connection "Upgrade";
#proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
proxy_next_upstream error timeout invalid_header http_502 http_503 http_504;
proxy_redirect off;
proxy_buffering off;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
client_body_timeout 45m;
proxy_read_timeout 45m;
}
}
I've change the timeouts, altered the proxy_set_header Connection $connection_upgrade; line and added the proxy_ssl_server_name on; . Still getting the 104: Unknown error. Still can't understand why it looks like the application returns a 200 OK, but nginx sees an error. Any ideas?
