maximum size of http headers in relayd

Previous Topic Next Topic
 
classic Classic list List threaded Threaded
6 messages Options
Reply | Threaded
Open this post in threaded view
|

maximum size of http headers in relayd

Sebastian Benoit-3
Currently relayd limits the size of all http headers to
RELAY_MAXHEADERLENGTH = 8192.

Current webbrowsers support for example as default

httpd: 8k
Apache: 8190 byte per header line, 100 header
nginx: 32k (max 8k per line)
iis: 16k per line
Tomcat: 8k

All except httpd are configurable and headers larger than our 8k limit are
encountered, especially as Apache has the 8k limit per line.

The following diff adds a new option for (http) protocols

    http protocol foo {
        http { headerlen 16000 }
    }

to make the maximum of all headers combined configurable.
It is still limited to a maximum of 128k, and i leave the default at the
current 8k.

Unfortunatly the choice of "http { }" requires a few mechanical parser
changes (because http is expected as a string in two places), and i'm happy
to do that differently if someone suggests another word.

comments, oks?

[1] https://httpd.apache.org/docs/2.4/mod/core.html#limitrequestfieldsize
[2] http://nginx.org/en/docs/http/ngx_http_core_module.html#large_client_header_buffers
[3] http://svn.apache.org/repos/asf/tomcat/trunk/java/org/apache/coyote/http11/AbstractHttp11Protocol.java



diff --git usr.sbin/relayd/parse.y usr.sbin/relayd/parse.y
index 5e357bb7eb2..86dee22fcd9 100644
--- usr.sbin/relayd/parse.y
+++ usr.sbin/relayd/parse.y
@@ -164,7 +164,7 @@ typedef struct {
 
 %token ALL APPEND BACKLOG BACKUP BUFFER CA CACHE SET CHECK CIPHERS CODE
 %token COOKIE DEMOTE DIGEST DISABLE ERROR EXPECT PASS BLOCK EXTERNAL FILENAME
-%token FORWARD FROM HASH HEADER HOST ICMP INCLUDE INET INET6 INTERFACE
+%token FORWARD FROM HASH HEADER HEADERLEN HOST HTTP ICMP INCLUDE INET INET6 INTERFACE
 %token INTERVAL IP LABEL LISTEN VALUE LOADBALANCE LOG LOOKUP METHOD MODE NAT
 %token NO DESTINATION NODELAY NOTHING ON PARENT PATH PFTAG PORT PREFORK
 %token PRIORITY PROTO QUERYSTR REAL REDIRECT RELAY REMOVE REQUEST RESPONSE
@@ -237,11 +237,10 @@ opttlsclient : /*empty*/ { $$ = 0; }
  | WITH ssltls { $$ = 1; }
  ;
 
-http_type : STRING {
+http_type : HTTP { $$ = 0; }
+ | STRING {
  if (strcmp("https", $1) == 0) {
  $$ = 1;
- } else if (strcmp("http", $1) == 0) {
- $$ = 0;
  } else {
  yyerror("invalid check type: %s", $1);
  free($1);
@@ -265,10 +264,9 @@ hostname : /* empty */ {
 
 relay_proto : /* empty */ { $$ = RELAY_PROTO_TCP; }
  | TCP { $$ = RELAY_PROTO_TCP; }
+ | HTTP { $$ = RELAY_PROTO_HTTP; }
  | STRING {
- if (strcmp("http", $1) == 0) {
- $$ = RELAY_PROTO_HTTP;
- } else if (strcmp("dns", $1) == 0) {
+ if (strcmp("dns", $1) == 0) {
  $$ = RELAY_PROTO_DNS;
  } else {
  yyerror("invalid protocol type: %s", $1);
@@ -311,7 +309,15 @@ eflags : STYLE STRING
  }
  ;
 
-port : PORT STRING {
+port : PORT HTTP {
+ int p = 0;
+ $$.op = PF_OP_EQ;
+ if ((p = getservice("http")) == -1)
+ YYERROR;
+ $$.val[0] = p;
+ $$.val[1] = 0;
+ }
+ | PORT STRING {
  char *a, *b;
  int p[2];
 
@@ -996,6 +1002,7 @@ proto : relay_proto PROTO STRING {
  p->tcpflags = TCPFLAG_DEFAULT;
  p->tlsflags = TLSFLAG_DEFAULT;
  p->tcpbacklog = RELAY_BACKLOG;
+ p->httpheaderlen = RELAY_DEFHEADERLENGTH;
  TAILQ_INIT(&p->rules);
  (void)strlcpy(p->tlsciphers, TLSCIPHERS_DEFAULT,
     sizeof(p->tlsciphers));
@@ -1034,12 +1041,28 @@ protoptsl : ssltls tlsflags
  | ssltls '{' tlsflags_l '}'
  | TCP tcpflags
  | TCP '{' tcpflags_l '}'
+ | HTTP httpflags
+ | HTTP '{' httpflags_l '}'
  | RETURN ERROR opteflags { proto->flags |= F_RETURN; }
  | RETURN ERROR '{' eflags_l '}' { proto->flags |= F_RETURN; }
  | filterrule
  | include
  ;
 
+
+httpflags_l : httpflags comma httpflags_l
+ | httpflags
+ ;
+
+httpflags : HEADERLEN NUMBER {
+ if ($2 < 0 || $2 > RELAY_MAXHEADERLENGTH) {
+ yyerror("invalid headerlen: %d", $2);
+ YYERROR;
+ }
+ proto->httpheaderlen = $2;
+ }
+ ;
+
 tcpflags_l : tcpflags comma tcpflags_l
  | tcpflags
  ;
@@ -2210,7 +2233,9 @@ lookup(char *s)
  { "from", FROM },
  { "hash", HASH },
  { "header", HEADER },
+ { "headerlen", HEADERLEN },
  { "host", HOST },
+ { "http", HTTP },
  { "icmp", ICMP },
  { "include", INCLUDE },
  { "inet", INET },
diff --git usr.sbin/relayd/relay_http.c usr.sbin/relayd/relay_http.c
index 2bd6952bbeb..26b560a55ab 100644
--- usr.sbin/relayd/relay_http.c
+++ usr.sbin/relayd/relay_http.c
@@ -187,9 +187,9 @@ relay_read_http(struct bufferevent *bev, void *arg)
 
  /* Limit the total header length minus \r\n */
  cre->headerlen += linelen;
- if (cre->headerlen > RELAY_MAXHEADERLENGTH) {
+ if (cre->headerlen > proto->httpheaderlen) {
  free(line);
- relay_abort_http(con, 413, "request too large", 0);
+ relay_abort_http(con, 413, "request headers too large", 0);
  return;
  }
 
diff --git usr.sbin/relayd/relayd.conf.5 usr.sbin/relayd/relayd.conf.5
index f79b0800a74..fc4b4ced33f 100644
--- usr.sbin/relayd/relayd.conf.5
+++ usr.sbin/relayd/relayd.conf.5
@@ -991,6 +991,15 @@ Enable the TLSv1.1 protocol.
 The default is
 .Ic no tlsv1.1 .
 .El
+.It Ic http Ar option
+Set the HTTP options and session settings.
+This is only used if HTTP is enabled in the relay.
+Valid options are:
+.Bl -tag -width Ds
+.It Ic headerlen Ar number
+Set the maximum size of all HTTP headers in bytes.
+The default value is 8192 and it is limited to a maximum of 131072.
+.El
 .El
 .Sh FILTER RULES
 Relays have the ability to filter connections based
diff --git usr.sbin/relayd/relayd.h usr.sbin/relayd/relayd.h
index 6d1ed6e1b0a..d0d8e0af8be 100644
--- usr.sbin/relayd/relayd.h
+++ usr.sbin/relayd/relayd.h
@@ -73,7 +73,8 @@
 #define RELAY_CACHESIZE -1 /* use default size */
 #define RELAY_NUMPROC 3
 #define RELAY_MAXHOSTS 32
-#define RELAY_MAXHEADERLENGTH 8192
+#define RELAY_MAXHEADERLENGTH 131072
+#define RELAY_DEFHEADERLENGTH 8192
 #define RELAY_STATINTERVAL 60
 #define RELAY_BACKLOG 10
 #define RELAY_MAXLOOKUPLEVELS 5
@@ -698,6 +699,7 @@ struct protocol {
  int tcpbacklog;
  u_int8_t tcpipttl;
  u_int8_t tcpipminttl;
+ size_t httpheaderlen;
  u_int8_t tlsflags;
  char tlsciphers[768];
  char tlsdhparams[128];

Reply | Threaded
Open this post in threaded view
|

Re: maximum size of http headers in relayd

Sebastian Benoit-3
rediff, i fixed the two lines that were too long.

Sebastian Benoit([hidden email]) on 2017.11.14 18:01:26 +0100:

> Currently relayd limits the size of all http headers to
> RELAY_MAXHEADERLENGTH = 8192.
>
> Current webbrowsers support for example as default
>
> httpd: 8k
> Apache: 8190 byte per header line, 100 header
> nginx: 32k (max 8k per line)
> iis: 16k per line
> Tomcat: 8k
>
> All except httpd are configurable and headers larger than our 8k limit are
> encountered, especially as Apache has the 8k limit per line.
>
> The following diff adds a new option for (http) protocols
>
>     http protocol foo {
>         http { headerlen 16000 }
>     }
>
> to make the maximum of all headers combined configurable.
> It is still limited to a maximum of 128k, and i leave the default at the
> current 8k.
>
> Unfortunatly the choice of "http { }" requires a few mechanical parser
> changes (because http is expected as a string in two places), and i'm happy
> to do that differently if someone suggests another word.
>
> comments, oks?
>
> [1] https://httpd.apache.org/docs/2.4/mod/core.html#limitrequestfieldsize
> [2] http://nginx.org/en/docs/http/ngx_http_core_module.html#large_client_header_buffers
> [3] http://svn.apache.org/repos/asf/tomcat/trunk/java/org/apache/coyote/http11/AbstractHttp11Protocol.java


diff --git usr.sbin/relayd/parse.y usr.sbin/relayd/parse.y
index 5e357bb7eb2..c9862a155c9 100644
--- usr.sbin/relayd/parse.y
+++ usr.sbin/relayd/parse.y
@@ -164,13 +164,13 @@ typedef struct {
 
 %token ALL APPEND BACKLOG BACKUP BUFFER CA CACHE SET CHECK CIPHERS CODE
 %token COOKIE DEMOTE DIGEST DISABLE ERROR EXPECT PASS BLOCK EXTERNAL FILENAME
-%token FORWARD FROM HASH HEADER HOST ICMP INCLUDE INET INET6 INTERFACE
-%token INTERVAL IP LABEL LISTEN VALUE LOADBALANCE LOG LOOKUP METHOD MODE NAT
-%token NO DESTINATION NODELAY NOTHING ON PARENT PATH PFTAG PORT PREFORK
-%token PRIORITY PROTO QUERYSTR REAL REDIRECT RELAY REMOVE REQUEST RESPONSE
-%token RETRY QUICK RETURN ROUNDROBIN ROUTE SACK SCRIPT SEND SESSION SNMP
-%token SOCKET SPLICE SSL STICKYADDR STYLE TABLE TAG TAGGED TCP TIMEOUT TLS TO
-%token ROUTER RTLABEL TRANSPARENT TRAP UPDATES URL VIRTUAL WITH TTL RTABLE
+%token FORWARD FROM HASH HEADER HEADERLEN HOST HTTP ICMP INCLUDE INET INET6
+%token INTERFACE INTERVAL IP LABEL LISTEN VALUE LOADBALANCE LOG LOOKUP METHOD
+%token MODE NAT NO DESTINATION NODELAY NOTHING ON PARENT PATH PFTAG PORT
+%token PREFORK PRIORITY PROTO QUERYSTR REAL REDIRECT RELAY REMOVE REQUEST
+%token RESPONSE RETRY QUICK RETURN ROUNDROBIN ROUTE SACK SCRIPT SEND SESSION
+%token SNMP SOCKET SPLICE SSL STICKYADDR STYLE TABLE TAG TAGGED TCP TIMEOUT TLS
+%token TO ROUTER RTLABEL TRANSPARENT TRAP UPDATES URL VIRTUAL WITH TTL RTABLE
 %token MATCH PARAMS RANDOM LEASTSTATES SRCHASH KEY CERTIFICATE PASSWORD ECDH
 %token EDH CURVE TICKETS
 %token <v.string> STRING
@@ -237,11 +237,10 @@ opttlsclient : /*empty*/ { $$ = 0; }
  | WITH ssltls { $$ = 1; }
  ;
 
-http_type : STRING {
+http_type : HTTP { $$ = 0; }
+ | STRING {
  if (strcmp("https", $1) == 0) {
  $$ = 1;
- } else if (strcmp("http", $1) == 0) {
- $$ = 0;
  } else {
  yyerror("invalid check type: %s", $1);
  free($1);
@@ -265,10 +264,9 @@ hostname : /* empty */ {
 
 relay_proto : /* empty */ { $$ = RELAY_PROTO_TCP; }
  | TCP { $$ = RELAY_PROTO_TCP; }
+ | HTTP { $$ = RELAY_PROTO_HTTP; }
  | STRING {
- if (strcmp("http", $1) == 0) {
- $$ = RELAY_PROTO_HTTP;
- } else if (strcmp("dns", $1) == 0) {
+ if (strcmp("dns", $1) == 0) {
  $$ = RELAY_PROTO_DNS;
  } else {
  yyerror("invalid protocol type: %s", $1);
@@ -311,7 +309,15 @@ eflags : STYLE STRING
  }
  ;
 
-port : PORT STRING {
+port : PORT HTTP {
+ int p = 0;
+ $$.op = PF_OP_EQ;
+ if ((p = getservice("http")) == -1)
+ YYERROR;
+ $$.val[0] = p;
+ $$.val[1] = 0;
+ }
+ | PORT STRING {
  char *a, *b;
  int p[2];
 
@@ -996,6 +1002,7 @@ proto : relay_proto PROTO STRING {
  p->tcpflags = TCPFLAG_DEFAULT;
  p->tlsflags = TLSFLAG_DEFAULT;
  p->tcpbacklog = RELAY_BACKLOG;
+ p->httpheaderlen = RELAY_DEFHEADERLENGTH;
  TAILQ_INIT(&p->rules);
  (void)strlcpy(p->tlsciphers, TLSCIPHERS_DEFAULT,
     sizeof(p->tlsciphers));
@@ -1034,12 +1041,28 @@ protoptsl : ssltls tlsflags
  | ssltls '{' tlsflags_l '}'
  | TCP tcpflags
  | TCP '{' tcpflags_l '}'
+ | HTTP httpflags
+ | HTTP '{' httpflags_l '}'
  | RETURN ERROR opteflags { proto->flags |= F_RETURN; }
  | RETURN ERROR '{' eflags_l '}' { proto->flags |= F_RETURN; }
  | filterrule
  | include
  ;
 
+
+httpflags_l : httpflags comma httpflags_l
+ | httpflags
+ ;
+
+httpflags : HEADERLEN NUMBER {
+ if ($2 < 0 || $2 > RELAY_MAXHEADERLENGTH) {
+ yyerror("invalid headerlen: %d", $2);
+ YYERROR;
+ }
+ proto->httpheaderlen = $2;
+ }
+ ;
+
 tcpflags_l : tcpflags comma tcpflags_l
  | tcpflags
  ;
@@ -2210,7 +2233,9 @@ lookup(char *s)
  { "from", FROM },
  { "hash", HASH },
  { "header", HEADER },
+ { "headerlen", HEADERLEN },
  { "host", HOST },
+ { "http", HTTP },
  { "icmp", ICMP },
  { "include", INCLUDE },
  { "inet", INET },
diff --git usr.sbin/relayd/relay_http.c usr.sbin/relayd/relay_http.c
index 2bd6952bbeb..db47a612231 100644
--- usr.sbin/relayd/relay_http.c
+++ usr.sbin/relayd/relay_http.c
@@ -187,9 +187,10 @@ relay_read_http(struct bufferevent *bev, void *arg)
 
  /* Limit the total header length minus \r\n */
  cre->headerlen += linelen;
- if (cre->headerlen > RELAY_MAXHEADERLENGTH) {
+ if (cre->headerlen > proto->httpheaderlen) {
  free(line);
- relay_abort_http(con, 413, "request too large", 0);
+ relay_abort_http(con, 413,
+    "request headers too large", 0);
  return;
  }
 
diff --git usr.sbin/relayd/relayd.conf.5 usr.sbin/relayd/relayd.conf.5
index f79b0800a74..fc4b4ced33f 100644
--- usr.sbin/relayd/relayd.conf.5
+++ usr.sbin/relayd/relayd.conf.5
@@ -991,6 +991,15 @@ Enable the TLSv1.1 protocol.
 The default is
 .Ic no tlsv1.1 .
 .El
+.It Ic http Ar option
+Set the HTTP options and session settings.
+This is only used if HTTP is enabled in the relay.
+Valid options are:
+.Bl -tag -width Ds
+.It Ic headerlen Ar number
+Set the maximum size of all HTTP headers in bytes.
+The default value is 8192 and it is limited to a maximum of 131072.
+.El
 .El
 .Sh FILTER RULES
 Relays have the ability to filter connections based
diff --git usr.sbin/relayd/relayd.h usr.sbin/relayd/relayd.h
index 6d1ed6e1b0a..d0d8e0af8be 100644
--- usr.sbin/relayd/relayd.h
+++ usr.sbin/relayd/relayd.h
@@ -73,7 +73,8 @@
 #define RELAY_CACHESIZE -1 /* use default size */
 #define RELAY_NUMPROC 3
 #define RELAY_MAXHOSTS 32
-#define RELAY_MAXHEADERLENGTH 8192
+#define RELAY_MAXHEADERLENGTH 131072
+#define RELAY_DEFHEADERLENGTH 8192
 #define RELAY_STATINTERVAL 60
 #define RELAY_BACKLOG 10
 #define RELAY_MAXLOOKUPLEVELS 5
@@ -698,6 +699,7 @@ struct protocol {
  int tcpbacklog;
  u_int8_t tcpipttl;
  u_int8_t tcpipminttl;
+ size_t httpheaderlen;
  u_int8_t tlsflags;
  char tlsciphers[768];
  char tlsdhparams[128];

Reply | Threaded
Open this post in threaded view
|

Re: maximum size of http headers in relayd

Alexander Bluhm
On Tue, Nov 14, 2017 at 06:15:05PM +0100, Sebastian Benoit wrote:
> +.It Ic http Ar option
> +Set the HTTP options and session settings.
> +This is only used if HTTP is enabled in the relay.

Could we check that the http option is not used for non-http configs?
Then writing correct relayd.conf gets easier.

anyway OK bluhm@

Reply | Threaded
Open this post in threaded view
|

Re: maximum size of http headers in relayd

Sebastian Benoit-3
Alexander Bluhm([hidden email]) on 2017.11.15 18:08:07 +0100:
> On Tue, Nov 14, 2017 at 06:15:05PM +0100, Sebastian Benoit wrote:
> > +.It Ic http Ar option
> > +Set the HTTP options and session settings.
> > +This is only used if HTTP is enabled in the relay.
>
> Could we check that the http option is not used for non-http configs?
> Then writing correct relayd.conf gets easier.
>
> anyway OK bluhm@

Thanks, here is a diff on top of the last to check that.

If you manage to set the default headerlen on non http protocols, it does
not catch that, but i dont want to add another variable for that corner
case.

ok?

Index: parse.y
===================================================================
RCS file: /cvs/src/usr.sbin/relayd/parse.y,v
retrieving revision 1.217
diff -u -p -r1.217 parse.y
--- parse.y 15 Nov 2017 19:03:26 -0000 1.217
+++ parse.y 15 Nov 2017 19:10:11 -0000
@@ -1023,6 +1023,12 @@ proto : relay_proto PROTO STRING {
  yyerror("invalid TLS protocol");
  YYERROR;
  }
+ if ((proto->type != RELAY_PROTO_HTTP) &&
+    proto->httpheaderlen != RELAY_DEFHEADERLENGTH) {
+ yyerror("can set http options only for "
+    "http protocol");
+ YYERROR;
+ }
 
  TAILQ_INSERT_TAIL(conf->sc_protos, proto, entry);
  }
 

Reply | Threaded
Open this post in threaded view
|

Re: maximum size of http headers in relayd

Alexander Bluhm
On Wed, Nov 15, 2017 at 08:15:15PM +0100, Sebastian Benoit wrote:
> Thanks, here is a diff on top of the last to check that.
>
> If you manage to set the default headerlen on non http protocols, it does
> not catch that, but i dont want to add another variable for that corner
> case.

I think it is simpler to check when we parse the http options.

bluhm

Index: parse.y
===================================================================
RCS file: /data/mirror/openbsd/cvs/src/usr.sbin/relayd/parse.y,v
retrieving revision 1.217
diff -u -p -r1.217 parse.y
--- parse.y 15 Nov 2017 19:03:26 -0000 1.217
+++ parse.y 16 Nov 2017 12:57:46 -0000
@@ -1055,6 +1055,11 @@ httpflags_l : httpflags comma httpflags_
  ;
 
 httpflags : HEADERLEN NUMBER {
+ if (proto->type != RELAY_PROTO_HTTP) {
+ yyerror("can set http options only for "
+    "http protocol");
+ YYERROR;
+ }
  if ($2 < 0 || $2 > RELAY_MAXHEADERLENGTH) {
  yyerror("invalid headerlen: %d", $2);
  YYERROR;

Reply | Threaded
Open this post in threaded view
|

Re: maximum size of http headers in relayd

Sebastian Benoit-3
Alexander Bluhm([hidden email]) on 2017.11.16 14:05:43 +0100:
> On Wed, Nov 15, 2017 at 08:15:15PM +0100, Sebastian Benoit wrote:
> > Thanks, here is a diff on top of the last to check that.
> >
> > If you manage to set the default headerlen on non http protocols, it does
> > not catch that, but i dont want to add another variable for that corner
> > case.
>
> I think it is simpler to check when we parse the http options.

yes you are right, the type is already set at that point.

ok benno
 

> bluhm
>
> Index: parse.y
> ===================================================================
> RCS file: /data/mirror/openbsd/cvs/src/usr.sbin/relayd/parse.y,v
> retrieving revision 1.217
> diff -u -p -r1.217 parse.y
> --- parse.y 15 Nov 2017 19:03:26 -0000 1.217
> +++ parse.y 16 Nov 2017 12:57:46 -0000
> @@ -1055,6 +1055,11 @@ httpflags_l : httpflags comma httpflags_
>   ;
>  
>  httpflags : HEADERLEN NUMBER {
> + if (proto->type != RELAY_PROTO_HTTP) {
> + yyerror("can set http options only for "
> +    "http protocol");
> + YYERROR;
> + }
>   if ($2 < 0 || $2 > RELAY_MAXHEADERLENGTH) {
>   yyerror("invalid headerlen: %d", $2);
>   YYERROR;
>