Merge pull request #4 from chainstack/bug/fix-reading-chunked-data

Bug/fix reading chunked data

Author: zaslavskii

.github/workflows/test.yaml

@@ -0,0 +1,16 @@
+name: Test
+
+on: [pull_request]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Unit
+      run: |
+        make test
+    - name: E2E
+      run: |
+        make test_e2e

.travis.yml

@@ -0,0 +1,9 @@
+.PHONY: test test_e2e
+
+test:
+	docker-compose build test && docker-compose run --rm test
+	docker-compose down -t 0
+
+test_e2e:
+	docker-compose build test_e2e && docker-compose run --rm test_e2e
+	docker-compose down -t 0

Makefile

@@ -1,4 +1,4 @@
-[![Travis Build Status](https://travis-ci.org/refinitiv/ngx_http_websocket_stat_module.svg?branch=master)](https://travis-ci.org/refinitiv/ngx_http_websocket_stat_module.svg?branch=master)
+![Tests](https://github.com/github/docs/actions/workflows/test.yml/badge.svg)
 
 
 # NGINX module websocket connection and traffic statistics
@@ -9,31 +9,20 @@ Nginx module developed for logging and displaying statistic of websocket proxy c
 
    1. Configure nginx adding this module with or build this module as a dynamic module:
    ```sh
-          ./configure (...) --add-module=./ngx_http_websocket_stat_module
+          ./configure (...) --add-module=./src/ngx_http_websocket_stat_module
           # or
-          ./configure (...) --add-dynamic-module=./ngx_http_websocket_stat_module && make modules
+          ./configure (...) --add-dynamic-module=./src/ngx_http_websocket_stat_module && make modules
    ```
-   2. Build nginx with make -j<n> command where n is number of cpu cores on your build machine
-
-   Alternatively could be used build script shipped along with module:
-   From module directory run
-   ```sh
-   test/build_helper.py build
-   ```
-   It would download and build nginx and all required libraries (openssl, pcre and zlib) and generate nginx configuration file.
 
 ## Usage
 
 To enable websocket frames logging specify `log_enabled on` and `ws_log_format` in server section of nginx config file. Additionally, specify `ws_log` to override the log file, which is used to log ws frames.
 
 To customize connection open and close log messages use "open" and "close" parameter for ws_log_format directive. 
-To log only when the full message is received/sent use "message" parameter for ws_log_format directive.
-
-Maximum number of concurrent websocket connections could be specified with ws_max_connections on server section. This value applies to whole connections that are on nginx. Argument should be integer representing maximum connections. When client tries to open more connections it recevies close framee with 1013 error code and connection is closed on nginx side. If zero number of connections is given there would be no limit on websocket connections.
+To log only when the full message is received/sent use "server"/"client" parameters for ws_log_format directive.
 
 To set maximum single connection lifetime use ws_conn_age parameter. Argument is time given in nginx time format (e.g. 1s, 1m 1h and so on). When connection's lifetime is exceeding specified value there is close websocket packet with 4001 error code generated and connection is closed.
 
-
 Here is a list of variables you can use in log format string:
 
  * $ws_opcode - websocket packet opcode. Look into https://tools.ietf.org/html/rfc6455 Section 5.2, Base Framing Protocol.
@@ -52,22 +41,7 @@ Here is a list of variables you can use in log format string:
  * $server_port - Server's port
  * $upstream_addr - websocket backend address
 
-## Example of configuration
-
-```
-
-server
-{
-   ws_log <path/to/logfile>;
-   ws_log_format "$time_local: packet of type $ws_opcode received from $ws_packet_source, packet size is $ws_payload_size";
-   ws_log_format open "$time_local: Connection opened";
-   ws_log_format close "$time_local: Connection closed";
-   ws_max_connections 200;
-   ws_conn_age 12h;
-   ...
-}
-
-```
+See [nginx sample configuraion](docker/nginx.conf).
 
 ## Copyright
 

README.md

@@ -0,0 +1,30 @@
+version: "3.6"
+services:
+    nginx:
+        command: nginx -g "daemon off;"
+        build:
+            context: .
+            dockerfile: ./docker/Dockerfile.nginx
+        volumes:
+            - ./docker/nginx.conf:/etc/nginx/nginx.conf
+            - logs:/var/log/nginx
+ 
+    test_e2e:
+        build:
+            context: .
+            dockerfile: ./docker/Dockerfile.e2e
+        environment:
+            - ENDPOINT=nginx
+            - LOG_FILE_PATH=/var/log/nginx/ws.log
+        volumes:
+            - logs:/var/log/nginx
+        depends_on:
+            - nginx
+
+    test:
+        build:
+            context: .
+            dockerfile: ./docker/Dockerfile.unit
+
+volumes:
+    logs:

docker-compose.yaml

@@ -0,0 +1,10 @@
+FROM rust:alpine3.13
+
+USER root
+
+RUN apk add --no-cache musl-dev && cargo install websocat
+
+COPY ./docker/test_e2e.sh /usr/local/bin/test_e2e
+RUN chmod +x /usr/local/bin/test_e2e
+
+CMD ["test_e2e"]

docker/Dockerfile.e2e

@@ -0,0 +1,18 @@
+FROM k8s.gcr.io/ingress-nginx/controller:v0.44.0 as builder
+
+USER root
+
+WORKDIR /tmp
+
+RUN apk add git openssl-dev pcre-dev zlib-dev libc-dev gcc make
+RUN NGINX_VERSION=$(nginx -v 2>&1 | sed 's/nginx version: nginx\///') && \
+    wget -qO- https://nginx.org/download/nginx-${NGINX_VERSION}.tar.gz | tar xvz && \
+    mv nginx-${NGINX_VERSION} nginx
+COPY ./src /tmp/ngx_http_websocket_stat_module
+RUN cd ./nginx && \
+    ./configure  --with-compat \
+    --add-dynamic-module=../ngx_http_websocket_stat_module && \
+    make modules
+
+FROM k8s.gcr.io/ingress-nginx/controller:v0.44.0
+COPY --from=builder /tmp/nginx/objs/ngx_http_websocket_stat_module.so /etc/nginx/modules/

docker/Dockerfile.nginx

@@ -0,0 +1,23 @@
+FROM alpine:3.10
+
+USER root
+
+RUN apk add gcc check-dev musl-dev
+
+COPY ./src /tmp/ngx_http_websocket_stat_format/src
+COPY ./tests /tmp/ngx_http_websocket_stat_format/tests
+
+WORKDIR /tmp/ngx_http_websocket_stat_format
+
+ENV SRC_DIR="./src" \
+    DIST_DIR="./dist" \
+    TEST_DIR="./tests" \
+    TEST_DIST_DIR="./tests/dist"
+
+RUN mkdir ${DIST_DIR} ${TEST_DIST_DIR} && \
+    gcc -g -DTEST -g -c ${SRC_DIR}/ngx_http_websocket_stat_format.c -o ${DIST_DIR}/ngx_http_websocket_stat_format.o && \
+	gcc -g -DTEST ${TEST_DIR}/test_format.c -c -o ${TEST_DIST_DIR}/test_format.o -lcheck && \
+	gcc -g -DTEST ${TEST_DIST_DIR}/test_format.o ${DIST_DIR}/ngx_http_websocket_stat_format.o -lcheck -lrt -o ${TEST_DIST_DIR}/test_format && \
+	cp ${TEST_DIST_DIR}/test_format /usr/local/bin/ngx_http_websocket_stat_test && chmod +x /usr/local/bin/ngx_http_websocket_stat_test
+
+CMD ["ngx_http_websocket_stat_test"]

docker/Dockerfile.unit

@@ -0,0 +1,24 @@
+pid /tmp/nginx.pid;
+
+load_module /etc/nginx/modules/ngx_http_websocket_stat_module.so;
+events {}
+http {
+  server {
+    listen 0.0.0.0:80;
+    ws_log /var/log/nginx/ws.log;
+    ws_log_format open "OPEN";
+    ws_log_format server "SERVER->CLIENT: $ws_message_size ($http_upgrade)";
+    ws_log_format client "CLIENT->SERVER: $ws_message_size ($http_upgrade)";
+    ws_log_format close "CLOSE";
+    ws_log_enabled on;
+    location / {
+      proxy_buffering off;
+      proxy_request_buffering off;
+      proxy_pass https://echo.websocket.org;
+      proxy_ssl_server_name on;
+      proxy_http_version 1.1;
+      proxy_set_header Connection "upgrade";
+      proxy_set_header Upgrade $http_upgrade;
+    }
+  }
+}

docker/nginx.conf

@@ -0,0 +1,18 @@
+#!/usr/bin/env sh
+
+set -exo pipefail
+
+> ${LOG_FILE_PATH}
+
+cat <<EOF > /tmp/ws-expected.log
+OPEN
+CLIENT->SERVER: 1311 (websocket)
+SERVER->CLIENT: 1311 (websocket)
+CLOSE
+EOF
+
+echo "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore et dolore magnam aliquam quaerat voluptatem. Ut enim ad minima veniam, quis nostrum exercitationem ullam corporis suscipit laboriosam, nisi ut aliquid ex ea commodi consequatur? Quis autem vel eum iure reprehenderit qui in ea voluptate velit esse quam nihil molestiae consequatur, vel illum qui dolorem eum fugiat quo voluptas nulla pariatur?" \
+| tr -d \\n \
+| websocat --text -1 ws://${ENDPOINT}
+
+diff ${LOG_FILE_PATH} /tmp/ws-expected.log