From 8a3241d5c46e8b8fac382f606147285e314f1f6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Mon, 2 Mar 2015 08:49:21 +0100 Subject: [PATCH 01/79] Initial commit Not even the demo is able to compile yet. I'm just tracking my progress. --- .gitignore | 9 + .gitmodules | 3 + .travis.yml | 21 + CMakeLists.txt | 83 ++++ LICENSE | 14 + README | 57 +++ cmake/FindLibEV.cmake | 18 + config.h.in | 8 + demo-json-rpc-server.c | 1065 ++++++++++++++++++++++++++++++++++++++++ liberty | 1 + 10 files changed, 1279 insertions(+) create mode 100644 .gitignore create mode 100644 .gitmodules create mode 100644 .travis.yml create mode 100644 CMakeLists.txt create mode 100644 LICENSE create mode 100644 README create mode 100644 cmake/FindLibEV.cmake create mode 100644 config.h.in create mode 100644 demo-json-rpc-server.c create mode 160000 liberty diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..465d9dd --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +# Build files +/build + +# Qt Creator files +/CMakeLists.txt.user* +/acid.config +/acid.files +/acid.creator* +/acid.includes diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..5abb60c --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "liberty"] + path = liberty + url = git://github.com/pjanouch/liberty.git diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..d8e73d0 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,21 @@ +language: c +notifications: + irc: + channels: "anathema.us.nu#anathema" + use_notice: true + skip_join: true +compiler: + - clang + - gcc +before_install: + - sudo add-apt-repository ppa:ukplc-team/ppa -y + - sudo apt-get update -qq +install: + - sudo apt-get install -y help2man libjansson-dev libev-dev +before_script: + - mkdir build + - cd build +script: + - cmake .. -DCMAKE_INSTALL_PREFIX=/usr + - make + - cpack -G DEB diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..ca9c0ea --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,83 @@ +project (acid C) +cmake_minimum_required (VERSION 2.8.5) + +# Moar warnings +if ("${CMAKE_C_COMPILER_ID}" MATCHES "GNU" OR CMAKE_COMPILER_IS_GNUC) + # -Wunused-function is pretty annoying here, as everything is static + set (CMAKE_C_FLAGS "-std=c99 -Wall -Wextra -Wno-unused-function") +endif ("${CMAKE_C_COMPILER_ID}" MATCHES "GNU" OR CMAKE_COMPILER_IS_GNUC) + +# Version +set (project_VERSION_MAJOR "0") +set (project_VERSION_MINOR "1") +set (project_VERSION_PATCH "0") + +set (project_VERSION "${project_VERSION_MAJOR}") +set (project_VERSION "${project_VERSION}.${project_VERSION_MINOR}") +set (project_VERSION "${project_VERSION}.${project_VERSION_PATCH}") + +# For custom modules +set (CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake) + +# Dependencies +find_package (PkgConfig REQUIRED) +pkg_check_modules (dependencies REQUIRED jansson) +find_package (LibEV REQUIRED) + +set (project_libraries ${dependencies_LIBRARIES} ${LIBEV_LIBRARIES}) +include_directories (${dependencies_INCLUDE_DIRS} ${LIBEV_INCLUDE_DIRS}) + +# Generate a configuration file +configure_file (${PROJECT_SOURCE_DIR}/config.h.in ${PROJECT_BINARY_DIR}/config.h) +include_directories (${PROJECT_BINARY_DIR}) + +# Build the executables +add_executable (demo-json-rpc-server demo-json-rpc-server.c) +target_link_libraries (demo-json-rpc-server ${project_libraries}) + +# The files to be installed +include (GNUInstallDirs) +install (TARGETS DESTINATION ${CMAKE_INSTALL_BINDIR}) +install (FILES LICENSE DESTINATION ${CMAKE_INSTALL_DOCDIR}) + +# Generate documentation from program help +find_program (HELP2MAN_EXECUTABLE help2man) +if (NOT HELP2MAN_EXECUTABLE) + message (FATAL_ERROR "help2man not found") +endif (NOT HELP2MAN_EXECUTABLE) + +foreach (page) + set (page_output "${PROJECT_BINARY_DIR}/${page}.1") + list (APPEND project_MAN_PAGES "${page_output}") + add_custom_command (OUTPUT ${page_output} + COMMAND ${HELP2MAN_EXECUTABLE} -N + "${PROJECT_BINARY_DIR}/${page}" -o ${page_output} + DEPENDS ${page} + COMMENT "Generating man page for ${page}" VERBATIM) +endforeach (page) + +add_custom_target (docs ALL DEPENDS ${project_MAN_PAGES}) + +foreach (page ${project_MAN_PAGES}) + string (REGEX MATCH "\\.([0-9])" manpage_suffix "${page}") + install (FILES "${page}" + DESTINATION "${CMAKE_INSTALL_MANDIR}/man${CMAKE_MATCH_1}") +endforeach (page) + +# CPack +set (CPACK_PACKAGE_DESCRIPTION_SUMMARY "A Continuous Integration Daemon") +set (CPACK_PACKAGE_VENDOR "Premysl Janouch") +set (CPACK_PACKAGE_CONTACT "Přemysl Janouch ") +set (CPACK_RESOURCE_FILE_LICENSE "${PROJECT_SOURCE_DIR}/LICENSE") +set (CPACK_PACKAGE_VERSION_MAJOR ${project_VERSION_MAJOR}) +set (CPACK_PACKAGE_VERSION_MINOR ${project_VERSION_MINOR}) +set (CPACK_PACKAGE_VERSION_PATCH ${project_VERSION_PATCH}) +set (CPACK_GENERATOR "TGZ;ZIP") +set (CPACK_PACKAGE_FILE_NAME + "${PROJECT_NAME}-${project_VERSION}-${CMAKE_SYSTEM_NAME}-${CMAKE_SYSTEM_PROCESSOR}") +set (CPACK_PACKAGE_INSTALL_DIRECTORY "${PROJECT_NAME}-${project_VERSION}") +set (CPACK_SOURCE_GENERATOR "TGZ;ZIP") +set (CPACK_SOURCE_IGNORE_FILES "/\\\\.git;/build;/CMakeLists.txt.user") +set (CPACK_SOURCE_PACKAGE_FILE_NAME "${PROJECT_NAME}-${project_VERSION}") + +include (CPack) diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..553e875 --- /dev/null +++ b/LICENSE @@ -0,0 +1,14 @@ + Copyright (c) 2015, Přemysl Janouch + All rights reserved. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY + SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION + OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN + CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/README b/README new file mode 100644 index 0000000..311e388 --- /dev/null +++ b/README @@ -0,0 +1,57 @@ +acid +==== + +`acid' is A Continuous Integration Daemon. Currently under heavy development. + +The aim of this project is to provide a dumbed-down alternative to Travis CI. +I find it way too complex to set up and run in a local setting, while the basic +gist of it is actually very simple -- run some stuff on new git commits. + +`acid' will provide a JSON-RPC 2.0 service for frontends over FastCGI or SCGI, +as well as a webhook endpoint for notifications about new commits. + +`acid' will be able to tell you about build results via e-mail and/or IRC. + +Builds will only be supported on the same machine as the daemon. Eventually I +might be able to add support for fully replicable builds using Docker. + +With this being my own project, of course it is written in event-looped C99 +where everything is stuffed into just a few files. At least I hope it's written +in a somewhat clean manner. Feel free to contribute. + +Building and Installing +----------------------- +Build dependencies: CMake, pkg-config, help2man, liberty (included) +Runtime dependencies: libev, Jansson + + $ git clone https://github.com/pjanouch/acid.git + $ git submodule init + $ git submodule update + $ mkdir build + $ cd build + $ cmake .. -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Debug + $ make + +To install the application, you can do either the usual: + # make install + +Or you can try telling CMake to make a package for you. For Debian it is: + $ cpack -G DEB + # dpkg -i acid-*.deb + +Note that for versions of CMake before 2.8.9, you need to prefix cpack with +`fakeroot' or file ownership will end up wrong. + +Usage +----- +TODO. The main application hasn't been written yet. + +License +------- +`acid' is written by Přemysl Janouch . + +You may use the software under the terms of the ISC license, the text of which +is included within the package, or, at your option, you may relicense the work +under the MIT or the Modified BSD License, as listed at the following site: + +http://www.gnu.org/licenses/license-list.html diff --git a/cmake/FindLibEV.cmake b/cmake/FindLibEV.cmake new file mode 100644 index 0000000..73787a1 --- /dev/null +++ b/cmake/FindLibEV.cmake @@ -0,0 +1,18 @@ +# Public Domain + +# The author of libev is a dick and doesn't want to add support for pkg-config, +# forcing us to include this pointless file in the distribution. + +# Some distributions do add it, though +find_package (PkgConfig REQUIRED) +pkg_check_modules (LIBEV QUIET libev) + +if (NOT LIBEV_FOUND) + find_path (LIBEV_INCLUDE_DIRS ev.h) + find_library (LIBEV_LIBRARIES NAMES ev) + + if (LIBEV_INCLUDE_DIRS AND LIBEV_LIBRARIES) + set (LIBEV_FOUND TRUE) + endif (LIBEV_INCLUDE_DIRS AND LIBEV_LIBRARIES) +endif (NOT LIBEV_FOUND) + diff --git a/config.h.in b/config.h.in new file mode 100644 index 0000000..89cd306 --- /dev/null +++ b/config.h.in @@ -0,0 +1,8 @@ +#ifndef CONFIG_H +#define CONFIG_H + +#define PROGRAM_NAME "${PROJECT_NAME}" +#define PROGRAM_VERSION "${project_VERSION}" + +#endif // ! CONFIG_H + diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c new file mode 100644 index 0000000..cbce78e --- /dev/null +++ b/demo-json-rpc-server.c @@ -0,0 +1,1065 @@ +/* + * demo-json-rpc-server.c: JSON-RPC 2.0 demo server + * + * Copyright (c) 2015, Přemysl Janouch + * All rights reserved. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY + * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION + * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN + * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + * + */ + +#define LIBERTY_WANT_SSL + +#define print_fatal_data ((void *) LOG_ERR) +#define print_error_data ((void *) LOG_ERR) +#define print_warning_data ((void *) LOG_WARNING) +#define print_status_data ((void *) LOG_INFO) +#define print_debug_data ((void *) LOG_DEBUG) + +#include "config.h" +#include "liberty/liberty.c" + +#include +#include +#include +#include + +#include +#include + +// --- Extensions to liberty --------------------------------------------------- + +// These should be incorporated into the library ASAP + +#define UNPACKER_INT_BEGIN \ + if (self->len - self->offset < sizeof *value) \ + return false; \ + uint8_t *x = (uint8_t *) self->data + self->offset; \ + self->offset += sizeof *value; + +static bool +msg_unpacker_u16 (struct msg_unpacker *self, uint16_t *value) +{ + UNPACKER_INT_BEGIN + *value + = (uint16_t) x[0] << 24 | (uint16_t) x[1] << 16; + return true; +} + +static bool +msg_unpacker_u32 (struct msg_unpacker *self, uint32_t *value) +{ + UNPACKER_INT_BEGIN + *value + = (uint32_t) x[0] << 24 | (uint32_t) x[1] << 16 + | (uint32_t) x[2] << 8 | (uint32_t) x[3]; + return true; +} + +#undef UNPACKER_INT_BEGIN + +// --- Logging ----------------------------------------------------------------- + +static void +log_message_syslog (void *user_data, const char *quote, const char *fmt, + va_list ap) +{ + int prio = (int) (intptr_t) user_data; + + va_list va; + va_copy (va, ap); + int size = vsnprintf (NULL, 0, fmt, va); + va_end (va); + if (size < 0) + return; + + char buf[size + 1]; + if (vsnprintf (buf, sizeof buf, fmt, ap) >= 0) + syslog (prio, "%s%s", quote, buf); +} + +// --- Configuration (application-specific) ------------------------------------ + +static struct config_item g_config_table[] = +{ + { "bind_host", NULL, "Address of the server" }, + { "port_fastcgi", "9000", "Port to bind for FastCGI" }, + { "port_scgi", NULL, "Port to bind for SCGI" }, + { NULL, NULL, NULL } +}; + +// --- FastCGI ----------------------------------------------------------------- + +// Constants from the FastCGI specification document + +#define FCGI_HEADER_LEN 8 + +#define FCGI_VERSION_1 1 +#define FCGI_NULL_REQUEST_ID 0 +#define FCGI_KEEP_CONN 1 + +enum fcgi_type +{ + FCGI_BEGIN_REQUEST = 1, + FCGI_ABORT_REQUEST = 2, + FCGI_END_REQUEST = 3, + FCGI_PARAMS = 4, + FCGI_STDIN = 5, + FCGI_STDOUT = 6, + FCGI_STDERR = 7, + FCGI_DATA = 8, + FCGI_GET_VALUES = 9, + FCGI_GET_VALUES_RESULT = 10, + FCGI_UNKNOWN_TYPE = 11, + FCGI_MAXTYPE = FCGI_UNKNOWN_TYPE +}; + +enum fcgi_role +{ + FCGI_RESPONDER = 1, + FCGI_AUTHORIZER = 2, + FCGI_FILTER = 3 +}; + +enum fcgi_protocol_status +{ + FCGI_REQUEST_COMPLETE = 0, + FCGI_CANT_MPX_CONN = 1, + FCGI_OVERLOADED = 2, + FCGI_UNKNOWN_ROLE = 3 +}; + +#define FCGI_MAX_CONNS "FCGI_MAX_CONNS" +#define FCGI_MAX_REQS "FCGI_MAX_REQS" +#define FCGI_MPXS_CONNS "FCGI_MPXS_CONNS" + +// - - Message stream parser - - - - - - - - - - - - - - - - - - - - - - - - - - + +struct fcgi_parser; + +typedef void (*fcgi_message_fn) + (const struct fcgi_parser *parser, void *user_data); + +enum fcgi_parser_state +{ + FCGI_READING_HEADER, ///< Reading the fixed header portion + FCGI_READING_CONTENT, ///< Reading the message content + FCGI_READING_PADDING ///< Reading the padding +}; + +struct fcgi_parser +{ + enum fcgi_parser_state state; ///< Parsing state + struct str input; ///< Input buffer + + // The next block of fields is considered public: + + uint8_t version; ///< FastCGI protocol version + uint8_t type; ///< FastCGI record type + uint16_t request_id; ///< FastCGI request ID + struct str content; ///< Message data + + uint16_t content_length; ///< Message content length + uint8_t padding_length; ///< Message padding length + + fcgi_message_fn on_message; ///< Callback on message + void *user_data; ///< User data +}; + +static void +fcgi_parser_init (struct fcgi_parser *self) +{ + memset (self, 0, sizeof *self); + str_init (&self->input); + str_init (&self->content); +} + +static void +fcgi_parser_free (struct fcgi_parser *self) +{ + str_free (&self->input); + str_free (&self->content); +} + +static void +fcgi_parser_unpack_header (struct fcgi_parser *self) +{ + struct msg_unpacker unpacker; + msg_unpacker_init (&unpacker, self->input.str, self->input.len); + + bool success = true; + uint8_t reserved; + success &= msg_unpacker_u8 (&unpacker, &self->version); + success &= msg_unpacker_u8 (&unpacker, &self->type); + success &= msg_unpacker_u16 (&unpacker, &self->request_id); + success &= msg_unpacker_u16 (&unpacker, &self->content_length); + success &= msg_unpacker_u8 (&unpacker, &self->padding_length); + success &= msg_unpacker_u8 (&unpacker, &reserved); + hard_assert (success); + + str_remove_slice (&self->input, 0, unpacker.offset); +} + +static void +fcgi_parser_push (struct fcgi_parser *self, const void *data, size_t len) +{ + // This could be made considerably faster for high-throughput applications + // if we use a circular buffer instead of constantly calling memmove() + str_append_data (&self->input, data, len); + + while (true) + switch (self->state) + { + case FCGI_READING_HEADER: + if (self->input.len < FCGI_HEADER_LEN) + return; + + fcgi_parser_unpack_header (self); + self->state = FCGI_READING_CONTENT; + break; + case FCGI_READING_CONTENT: + if (self->input.len < self->content_length) + return; + + // Move an appropriate part of the input buffer to the content buffer + str_reset (&self->content); + str_append_data (&self->content, self->input.str, self->content_length); + str_remove_slice (&self->input, 0, self->content_length); + self->state = FCGI_READING_PADDING; + break; + case FCGI_READING_PADDING: + if (self->input.len < self->padding_length) + return; + + // Call the callback to further process the message + self->on_message (self, self->user_data); + + // Remove the padding from the input buffer + str_remove_slice (&self->input, 0, self->padding_length); + self->state = FCGI_READING_HEADER; + break; + } +} + +// - - Name-value pair parser - - - - - - - - - - - - - - - - - - - - - - - - - + +enum fcgi_nv_parser_state +{ + FCGI_NV_PARSER_NAME_LEN, ///< The first name length octet + FCGI_NV_PARSER_NAME_LEN_FULL, ///< Remaining name length octets + FCGI_NV_PARSER_VALUE_LEN, ///< The first value length octet + FCGI_NV_PARSER_VALUE_LEN_FULL, ///< Remaining value length octets + FCGI_NV_PARSER_NAME, ///< Reading the name + FCGI_NV_PARSER_VALUE ///< Reading the value +}; + +struct fcgi_nv_parser +{ + struct str_map *output; ///< Where the pairs will be stored + + enum fcgi_nv_parser_state state; ///< Parsing state + struct str input; ///< Input buffer + + uint32_t name_len; ///< Length of the name + uint32_t value_len; ///< Length of the value + + char *name; ///< The current name, 0-terminated + char *value; ///< The current value, 0-terminated +}; + +static void +fcgi_nv_parser_init (struct fcgi_nv_parser *self) +{ + memset (self, 0, sizeof *self); + str_init (&self->input); +} + +static void +fcgi_nv_parser_free (struct fcgi_nv_parser *self) +{ + str_free (&self->input); + free (self->name); + free (self->value); +} + +static void +fcgi_nv_parser_push (struct fcgi_nv_parser *self, void *data, size_t len) +{ + // This could be optimized significantly; I'm not even trying + str_append_data (&self->input, data, len); + + while (true) + { + struct msg_unpacker unpacker; + msg_unpacker_init (&unpacker, self->input.str, self->input.len); + + switch (self->state) + { + uint8_t len; + uint32_t len_full; + + case FCGI_NV_PARSER_NAME_LEN: + if (!msg_unpacker_u8 (&unpacker, &len)) + return; + + if (len >> 7) + self->state = FCGI_NV_PARSER_NAME_LEN_FULL; + else + { + self->name_len = len; + str_remove_slice (&self->input, 0, unpacker.offset); + self->state = FCGI_NV_PARSER_VALUE_LEN; + } + break; + case FCGI_NV_PARSER_NAME_LEN_FULL: + if (!msg_unpacker_u32 (&unpacker, &len_full)) + return; + + self->name_len = len_full & ~(1U << 31); + str_remove_slice (&self->input, 0, unpacker.offset); + self->state = FCGI_NV_PARSER_VALUE_LEN; + break; + case FCGI_NV_PARSER_VALUE_LEN: + if (!msg_unpacker_u8 (&unpacker, &len)) + return; + + if (len >> 7) + self->state = FCGI_NV_PARSER_VALUE_LEN_FULL; + else + { + self->value_len = len; + str_remove_slice (&self->input, 0, unpacker.offset); + self->state = FCGI_NV_PARSER_NAME; + } + break; + case FCGI_NV_PARSER_VALUE_LEN_FULL: + if (!msg_unpacker_u32 (&unpacker, &len_full)) + return; + + self->value_len = len_full & ~(1U << 31); + str_remove_slice (&self->input, 0, unpacker.offset); + self->state = FCGI_NV_PARSER_NAME; + break; + case FCGI_NV_PARSER_NAME: + if (self->input.len < self->name_len) + return; + + self->name = xmalloc (self->name_len + 1); + self->name[self->name_len] = '\0'; + memcpy (self->name, self->input.str, self->name_len); + str_remove_slice (&self->input, 0, self->name_len); + self->state = FCGI_NV_PARSER_VALUE; + break; + case FCGI_NV_PARSER_VALUE: + if (self->input.len < self->value_len) + return; + + self->value = xmalloc (self->value_len + 1); + self->value[self->value_len] = '\0'; + memcpy (self->value, self->input.str, self->value_len); + str_remove_slice (&self->input, 0, self->value_len); + self->state = FCGI_NV_PARSER_NAME_LEN; + + // The map takes ownership of the value + str_map_set (self->output, self->name, self->value); + free (self->name); + + self->name = NULL; + self->value = NULL; + break; + } + } +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +// TODO +struct fcgi_request +{ + struct fcgi_muxer *muxer; ///< The parent muxer + + uint16_t request_id; ///< The ID of this request +}; + +// TODO +struct fcgi_muxer +{ + struct fcgi_parser parser; ///< FastCGI message parser + + /// Requests assigned to request IDs + // TODO: allocate this dynamically + struct fcgi_request *requests[1 << 16]; +}; + +static void +fcgi_muxer_on_message (const struct fcgi_parser *parser, void *user_data) +{ + struct fcgi_muxer *self = user_data; + + // TODO +} + +static void +fcgi_muxer_init (struct fcgi_muxer *self) +{ + fcgi_parser_init (&self->parser); + self->parser.on_message = fcgi_muxer_on_message; + self->parser.user_data = self; +} + +static void +fcgi_muxer_free (struct fcgi_muxer *self) +{ + fcgi_parser_free (&self->parser); +} + +static void +fcgi_muxer_push (struct fcgi_muxer *self, const void *data, size_t len) +{ + fcgi_parser_push (&self->parser, data, len); +} + +// --- SCGI -------------------------------------------------------------------- + +enum scgi_parser_state +{ + SCGI_READING_NETSTRING_LENGTH, ///< The length of the header netstring + SCGI_READING_NAME, ///< Header name + SCGI_READING_VALUE, ///< Header value + SCGI_READING_CONTENT ///< Incoming data +}; + +struct scgi_parser +{ + enum scgi_parser_state state; ///< Parsing state + struct str input; ///< Input buffer + + struct str_map headers; ///< Headers parsed + + size_t headers_len; ///< Length of the netstring contents + struct str name; ///< Header name so far + struct str value; ///< Header value so far +}; + +static void +scgi_parser_init (struct scgi_parser *self) +{ + str_init (&self->input); + str_map_init (&self->headers); + self->headers.free = free; + str_init (&self->name); + str_init (&self->value); +} + +static void +scgi_parser_free (struct scgi_parser *self) +{ + str_free (&self->input); + str_map_free (&self->headers); + str_free (&self->name); + str_free (&self->value); +} + +static bool +scgi_parser_push (struct scgi_parser *self, + void *data, size_t len, struct error **e) +{ + // This retarded netstring madness is even more complicated than FastCGI; + // this procedure could also be optimized significantly + str_append_data (&self->input, data, len); + + while (true) + switch (self->state) + { + case SCGI_READING_NETSTRING_LENGTH: + { + if (self->input.len < 1) + return true; + + char digit = *self->input.str; + // XXX: this allows for omitting the netstring length altogether + if (digit == ':') + { + self->state = SCGI_READING_NAME; + break; + } + + if (digit < '0' || digit >= '9') + { + error_set (e, "invalid header netstring"); + return false; + } + + size_t new_len = self->headers_len * 10 + (digit - '0'); + if (new_len < self->headers_len) + { + error_set (e, "header netstring is too long"); + return false; + } + self->headers_len = new_len; + str_remove_slice (&self->input, 0, 1); + break; + } + case SCGI_READING_NAME: + { + if (self->input.len < 1) + return true; + + char c = *self->input.str; + if (!self->headers_len) + { + // The netstring is ending but we haven't finished parsing it, + // or the netstring doesn't end with a comma + if (self->name.len || c != ',') + { + error_set (e, "invalid header netstring"); + return false; + } + self->state = SCGI_READING_CONTENT; + } + else if (c != '\0') + str_append_c (&self->name, c); + else + self->state = SCGI_READING_VALUE; + + str_remove_slice (&self->input, 0, 1); + break; + } + case SCGI_READING_VALUE: + { + if (self->input.len < 1) + return true; + + char c = *self->input.str; + if (!self->headers_len) + { + // The netstring is ending but we haven't finished parsing it + error_set (e, "invalid header netstring"); + return false; + } + else if (c != '\0') + str_append_c (&self->value, c); + else + { + // We've got a name-value pair, let's put it in the map + str_map_set (&self->headers, + self->name.str, str_steal (&self->value)); + + str_reset (&self->name); + str_init (&self->value); + + self->state = SCGI_READING_NAME; + } + + str_remove_slice (&self->input, 0, 1); + break; + } + case SCGI_READING_CONTENT: + // TODO: I have no idea what to do with the contents + return true; + + break; + } +} + +// --- ? ----------------------------------------------------------------------- + +// TODO +struct client +{ + LIST_HEADER (struct client) + + struct server_context *ctx; ///< Server context + + int socket_fd; ///< The TCP socket + struct str read_buffer; ///< Unprocessed input + write_queue_t write_queue; ///< Write queue + + ev_io read_watcher; ///< The socket can be read from + ev_io write_watcher; ///< The socket can be written to +}; + +static void +client_init (struct client *self) +{ + memset (self, 0, sizeof *self); + str_init (&self->read_buffer); + write_queue_init (&self->write_queue); +} + +static void +client_free (struct client *self) +{ + str_free (&self->read_buffer); + write_queue_free (&self->write_queue); +} + +// --- ? ----------------------------------------------------------------------- + +enum listener_type +{ + LISTENER_FCGI, ///< FastCGI + LISTENER_SCGI ///< SCGI +}; + +struct listener +{ + int fd; ///< Listening socket FD + ev_io watcher; ///< New connection available + enum listener_type type; ///< The protocol +}; + +struct server_context +{ + ev_signal sigterm_watcher; ///< Got SIGTERM + ev_signal sigint_watcher; ///< Got SIGINT + bool quitting; ///< User requested quitting + + struct listener *listeners; ///< Listeners + size_t n_listeners; ///< Number of listening sockets + + struct client *clients; ///< Clients + unsigned n_clients; ///< Current number of connections + + struct str_map config; ///< Server configuration +}; + +static void +server_context_init (struct server_context *self) +{ + memset (self, 0, sizeof *self); + + str_map_init (&self->config); + load_config_defaults (&self->config, g_config_table); +} + +static void +server_context_free (struct server_context *self) +{ + // TODO: free the clients (?) + // TODO: close the listeners (?) + + str_map_free (&self->config); +} + +// --- ? ----------------------------------------------------------------------- + +static void +remove_client (struct server_context *ctx, struct client *client) +{ + ev_io_stop (EV_DEFAULT_ &client->read_watcher); + ev_io_stop (EV_DEFAULT_ &client->write_watcher); + xclose (client->socket_fd); + LIST_UNLINK (ctx->clients, client); + client_free (client); + free (client); +} + +static bool +read_loop (EV_P_ ev_io *watcher, + bool (*cb) (EV_P_ ev_io *, const void *, ssize_t)) +{ + char buf[8192]; + while (true) + { + ssize_t n_read = recv (watcher->fd, buf, sizeof buf, 0); + if (n_read < 0) + { + if (errno == EAGAIN) + break; + if (errno == EINTR) + continue; + } + if (n_read <= 0 || !cb (EV_A_ watcher, buf, n_read)) + return false; + } + return true; +} + +static bool +flush_queue (write_queue_t *queue, ev_io *watcher) +{ + struct iovec vec[queue->len], *vec_iter = vec; + for (write_req_t *iter = queue->head; iter; iter = iter->next) + *vec_iter++ = iter->data; + + ssize_t written; +again: + written = writev (watcher->fd, vec, N_ELEMENTS (vec)); + if (written < 0) + { + if (errno == EAGAIN) + goto skip; + if (errno == EINTR) + goto again; + return false; + } + + write_queue_processed (queue, written); + +skip: + if (write_queue_is_empty (queue)) + ev_io_stop (EV_DEFAULT_ watcher); + else + ev_io_start (EV_DEFAULT_ watcher); + return true; +} + +static void +on_client_ready (EV_P_ ev_io *watcher, int revents) +{ + struct server_context *ctx = ev_userdata (loop); + struct client *client = watcher->data; + + if (revents & EV_READ) + if (!read_loop (EV_A_ watcher, on_client_data)) + goto error; + if (revents & EV_WRITE) + if (!flush_queue (&client->write_queue, watcher)) + goto error; + return; + +error: + remove_client (ctx, client); +} + +static void +on_fcgi_client_available (EV_P_ ev_io *watcher, int revents) +{ + struct server_context *ctx = ev_userdata (loop); + (void) revents; + + // TODO + while (true) + { + int sock_fd = accept (watcher->fd, NULL, NULL); + if (sock_fd == -1) + { + if (errno == EAGAIN) + break; + if (errno == EINTR + || errno == ECONNABORTED) + continue; + + // Stop accepting connections to prevent busy looping + ev_io_stop (EV_A_ watcher); + + print_fatal ("%s: %s", "accept", strerror (errno)); + // TODO: initiate_quit (ctx); + break; + } + + struct client *client = xmalloc (sizeof *client); + client_init (client); + client->socket_fd = sock_fd; + + set_blocking (sock_fd, false); + ev_io_init (&client->read_watcher, on_client_ready, sock_fd, EV_READ); + ev_io_init (&client->write_watcher, on_client_ready, sock_fd, EV_WRITE); + client->read_watcher.data = client; + client->write_watcher.data = client; + + // We're only interested in reading as the write queue is empty now + ev_io_start (EV_A_ &client->read_watcher); + + LIST_PREPEND (ctx->clients, client); + ctx->n_clients++; + } +} + +// --- Application setup ------------------------------------------------------- + +/// This function handles values that require validation before their first use, +/// or some kind of a transformation (such as conversion to an integer) needs +/// to be done before they can be used directly. +static bool +parse_config (struct server_context *ctx, struct error **e) +{ + (void) ctx; + (void) e; + + return true; +} + +static int +listen_finish (struct addrinfo *gai_iter) +{ + int fd = socket (gai_iter->ai_family, + gai_iter->ai_socktype, gai_iter->ai_protocol); + if (fd == -1) + return -1; + set_cloexec (fd); + + int yes = 1; + soft_assert (setsockopt (fd, SOL_SOCKET, SO_KEEPALIVE, + &yes, sizeof yes) != -1); + soft_assert (setsockopt (fd, SOL_SOCKET, SO_REUSEADDR, + &yes, sizeof yes) != -1); + + char host[NI_MAXHOST], port[NI_MAXSERV]; + host[0] = port[0] = '\0'; + int err = getnameinfo (gai_iter->ai_addr, gai_iter->ai_addrlen, + host, sizeof host, port, sizeof port, + NI_NUMERICHOST | NI_NUMERICSERV); + if (err) + print_debug ("%s: %s", "getnameinfo", gai_strerror (err)); + + char *address = format_host_port_pair (host, port); + if (bind (fd, gai_iter->ai_addr, gai_iter->ai_addrlen)) + print_error ("bind to %s failed: %s", address, strerror (errno)); + else if (listen (fd, 16 /* arbitrary number */)) + print_error ("listen on %s failed: %s", address, strerror (errno)); + else + { + print_status ("listening on %s", address); + free (address); + return fd; + } + + free (address); + xclose (fd); + return -1; +} + +static void +listen_resolve (struct server_context *ctx, const char *host, const char *port, + struct addrinfo *gai_hints, enum listener_type type) +{ + struct addrinfo *gai_result, *gai_iter; + int err = getaddrinfo (host, port, gai_hints, &gai_result); + if (err) + { + char *address = format_host_port_pair (host, port); + print_error ("bind to %s failed: %s: %s", + address, "getaddrinfo", gai_strerror (err)); + free (address); + return; + } + + int fd; + for (gai_iter = gai_result; gai_iter; gai_iter = gai_iter->ai_next) + { + if ((fd = listen_finish (gai_iter)) == -1) + continue; + set_blocking (fd, false); + + struct listener *listener = &ctx->listeners[ctx->n_listeners++]; + switch ((listener->type = type)) + { + case LISTENER_FCGI: + ev_io_init (&listener->watcher, + on_fcgi_client_available, fd, EV_READ); + break; + case LISTENER_SCGI: + ev_io_init (&listener->watcher, + on_scgi_client_available, fd, EV_READ); + break; + } + + ev_io_start (EV_DEFAULT_ &listener->watcher); + break; + } + freeaddrinfo (gai_result); +} + +static bool +setup_listen_fds (struct server_context *ctx, struct error **e) +{ + const char *bind_host = str_map_find (&ctx->config, "bind_host"); + const char *port_fcgi = str_map_find (&ctx->config, "port_fastcgi"); + const char *port_scgi = str_map_find (&ctx->config, "port_scgi"); + + struct addrinfo gai_hints; + memset (&gai_hints, 0, sizeof gai_hints); + + gai_hints.ai_socktype = SOCK_STREAM; + gai_hints.ai_flags = AI_PASSIVE; + + struct str_vector ports_fcgi; + struct str_vector ports_scgi; + str_vector_init (&ports_fcgi); + str_vector_init (&ports_scgi); + + if (port_fcgi) + split_str_ignore_empty (port_fcgi, ',', &ports_fcgi); + if (port_scgi) + split_str_ignore_empty (port_scgi, ',', &ports_scgi); + + size_t n_ports = ports_fcgi.len + ports_scgi.len; + ctx->listeners = xcalloc (n_ports, sizeof *ctx->listeners); + + for (size_t i = 0; i < ports_fcgi.len; i++) + listen_resolve (ctx, bind_host, ports_fcgi.vector[i], + &gai_hints, LISTENER_FCGI); + for (size_t i = 0; i < ports_scgi.len; i++) + listen_resolve (ctx, bind_host, ports_scgi.vector[i], + &gai_hints, LISTENER_SCGI); + + str_vector_free (&ports_fcgi); + str_vector_free (&ports_scgi); + + if (!ctx->n_listeners) + { + error_set (e, "%s: %s", + "network setup failed", "no ports to listen on"); + return false; + } + return true; +} + +// --- Main program ------------------------------------------------------------ + +static void +on_termination_signal (EV_P_ ev_signal *handle, int revents) +{ + struct server_context *ctx = ev_userdata (loop); + (void) handle; + (void) revents; + + // TODO: initiate_quit (ctx); +} + +static void +daemonize (void) +{ + // TODO: create and lock a PID file? + print_status ("daemonizing..."); + + if (chdir ("/")) + exit_fatal ("%s: %s", "chdir", strerror (errno)); + + pid_t pid; + if ((pid = fork ()) < 0) + exit_fatal ("%s: %s", "fork", strerror (errno)); + else if (pid) + exit (EXIT_SUCCESS); + + setsid (); + signal (SIGHUP, SIG_IGN); + + if ((pid = fork ()) < 0) + exit_fatal ("%s: %s", "fork", strerror (errno)); + else if (pid) + exit (EXIT_SUCCESS); + + openlog (PROGRAM_NAME, LOG_NDELAY | LOG_NOWAIT | LOG_PID, 0); + g_log_message_real = log_message_syslog; + + // XXX: we may close our own descriptors this way, crippling ourselves + for (int i = 0; i < 3; i++) + xclose (i); + + int tty = open ("/dev/null", O_RDWR); + if (tty != 0 || dup (0) != 1 || dup (0) != 2) + exit_fatal ("failed to reopen FD's: %s", strerror (errno)); +} + +static void +parse_program_arguments (int argc, char **argv) +{ + static const struct opt opts[] = + { + { 'd', "debug", NULL, 0, "run in debug mode" }, + { 'h', "help", NULL, 0, "display this help and exit" }, + { 'V', "version", NULL, 0, "output version information and exit" }, + { 'w', "write-default-cfg", "FILENAME", + OPT_OPTIONAL_ARG | OPT_LONG_ONLY, + "write a default configuration file and exit" }, + { 0, NULL, NULL, 0, NULL } + }; + + struct opt_handler oh; + opt_handler_init (&oh, argc, argv, opts, NULL, "JSON-RPC 2.0 demo server."); + + int c; + while ((c = opt_handler_get (&oh)) != -1) + switch (c) + { + case 'd': + g_debug_mode = true; + break; + case 'h': + opt_handler_usage (&oh, stdout); + exit (EXIT_SUCCESS); + case 'V': + printf (PROGRAM_NAME " " PROGRAM_VERSION "\n"); + exit (EXIT_SUCCESS); + case 'w': + call_write_default_config (optarg, g_config_table); + exit (EXIT_SUCCESS); + default: + print_error ("wrong options"); + opt_handler_usage (&oh, stderr); + exit (EXIT_FAILURE); + } + + argc -= optind; + argv += optind; + + if (argc) + { + opt_handler_usage (&oh, stderr); + exit (EXIT_FAILURE); + } + opt_handler_free (&oh); +} + +int +main (int argc, char *argv[]) +{ + parse_program_arguments (argc, argv); + + print_status (PROGRAM_NAME " " PROGRAM_VERSION " starting"); + + struct server_context ctx; + server_context_init (&ctx); + + struct error *e = NULL; + if (!read_config_file (&ctx.config, &e)) + { + print_error ("error loading configuration: %s", e->message); + error_free (e); + exit (EXIT_FAILURE); + } + + struct ev_loop *loop; + if (!(loop = EV_DEFAULT)) + exit_fatal ("libev initialization failed"); + + ev_set_userdata (loop, &ctx); + + ev_signal_init (&ctx.sigterm_watcher, on_termination_signal, SIGTERM); + ev_signal_start (EV_DEFAULT_ &ctx.sigterm_watcher); + + ev_signal_init (&ctx.sigint_watcher, on_termination_signal, SIGINT); + ev_signal_start (EV_DEFAULT_ &ctx.sigint_watcher); + + (void) signal (SIGPIPE, SIG_IGN); + + if (!parse_config (&ctx, &e) + || !setup_listen_fds (&ctx, &e)) + { + print_error ("%s", e->message); + error_free (e); + exit (EXIT_FAILURE); + } + + if (!g_debug_mode) + daemonize (); + + ev_run (loop, 0); + ev_loop_destroy (loop); + + server_context_free (&ctx); + return EXIT_SUCCESS; +} diff --git a/liberty b/liberty new file mode 160000 index 0000000..0876458 --- /dev/null +++ b/liberty @@ -0,0 +1 @@ +Subproject commit 087645848baec5e59e4296817850bd5dd240cbb2 From a54230bddba09f68f4d7fda23f44962050df7a07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Mon, 2 Mar 2015 23:11:29 +0100 Subject: [PATCH 02/79] Steady progress I'm trying to figure out everything at once, i.e. the entire structure of the application from top to bottom, trying to converge on a workable design while refactoring still doesn't hurt as much as it would once it's established. --- demo-json-rpc-server.c | 428 ++++++++++++++++++++++++++++------------- 1 file changed, 296 insertions(+), 132 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index cbce78e..8645ae3 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -18,8 +18,6 @@ * */ -#define LIBERTY_WANT_SSL - #define print_fatal_data ((void *) LOG_ERR) #define print_error_data ((void *) LOG_ERR) #define print_warning_data ((void *) LOG_WARNING) @@ -68,6 +66,61 @@ msg_unpacker_u32 (struct msg_unpacker *self, uint32_t *value) #undef UNPACKER_INT_BEGIN +// --- libev helpers ----------------------------------------------------------- + +static bool +read_loop (EV_P_ ev_io *watcher, + bool (*cb) (EV_P_ ev_io *, const void *, ssize_t)) +{ + char buf[8192]; + while (true) + { + ssize_t n_read = recv (watcher->fd, buf, sizeof buf, 0); + if (n_read < 0) + { + if (errno == EAGAIN) + break; + if (errno == EINTR) + continue; + } + // The callback is called on EOF as well + if (n_read < 0 || !cb (EV_A_ watcher, buf, n_read)) + return false; + if (!n_read) + return false; + } + return true; +} + +static bool +flush_queue (write_queue_t *queue, ev_io *watcher) +{ + struct iovec vec[queue->len], *vec_iter = vec; + for (write_req_t *iter = queue->head; iter; iter = iter->next) + *vec_iter++ = iter->data; + + ssize_t written; +again: + written = writev (watcher->fd, vec, N_ELEMENTS (vec)); + if (written < 0) + { + if (errno == EAGAIN) + goto skip; + if (errno == EINTR) + goto again; + return false; + } + + write_queue_processed (queue, written); + +skip: + if (write_queue_is_empty (queue)) + ev_io_stop (EV_DEFAULT_ watcher); + else + ev_io_start (EV_DEFAULT_ watcher); + return true; +} + // --- Logging ----------------------------------------------------------------- static void @@ -88,16 +141,6 @@ log_message_syslog (void *user_data, const char *quote, const char *fmt, syslog (prio, "%s%s", quote, buf); } -// --- Configuration (application-specific) ------------------------------------ - -static struct config_item g_config_table[] = -{ - { "bind_host", NULL, "Address of the server" }, - { "port_fastcgi", "9000", "Port to bind for FastCGI" }, - { "port_scgi", NULL, "Port to bind for SCGI" }, - { NULL, NULL, NULL } -}; - // --- FastCGI ----------------------------------------------------------------- // Constants from the FastCGI specification document @@ -472,9 +515,21 @@ scgi_parser_free (struct scgi_parser *self) static bool scgi_parser_push (struct scgi_parser *self, - void *data, size_t len, struct error **e) + const void *data, size_t len, struct error **e) { - // This retarded netstring madness is even more complicated than FastCGI; + if (!len) + { + if (self->state != SCGI_READING_CONTENT) + { + error_set (e, "premature EOF"); + return false; + } + + // TODO: a "on_eof" callback? + return true; + } + + // Notice that this madness is significantly harder to parse than FastCGI; // this procedure could also be optimized significantly str_append_data (&self->input, data, len); @@ -526,6 +581,7 @@ scgi_parser_push (struct scgi_parser *self, return false; } self->state = SCGI_READING_CONTENT; + // TODO: a "on_headers_read" callback? } else if (c != '\0') str_append_c (&self->name, c); @@ -565,58 +621,21 @@ scgi_parser_push (struct scgi_parser *self, break; } case SCGI_READING_CONTENT: - // TODO: I have no idea what to do with the contents + // TODO: a "on_content" callback? return true; break; } } -// --- ? ----------------------------------------------------------------------- +// --- Server ------------------------------------------------------------------ -// TODO -struct client +static struct config_item g_config_table[] = { - LIST_HEADER (struct client) - - struct server_context *ctx; ///< Server context - - int socket_fd; ///< The TCP socket - struct str read_buffer; ///< Unprocessed input - write_queue_t write_queue; ///< Write queue - - ev_io read_watcher; ///< The socket can be read from - ev_io write_watcher; ///< The socket can be written to -}; - -static void -client_init (struct client *self) -{ - memset (self, 0, sizeof *self); - str_init (&self->read_buffer); - write_queue_init (&self->write_queue); -} - -static void -client_free (struct client *self) -{ - str_free (&self->read_buffer); - write_queue_free (&self->write_queue); -} - -// --- ? ----------------------------------------------------------------------- - -enum listener_type -{ - LISTENER_FCGI, ///< FastCGI - LISTENER_SCGI ///< SCGI -}; - -struct listener -{ - int fd; ///< Listening socket FD - ev_io watcher; ///< New connection available - enum listener_type type; ///< The protocol + { "bind_host", NULL, "Address of the server" }, + { "port_fastcgi", "9000", "Port to bind for FastCGI" }, + { "port_scgi", NULL, "Port to bind for SCGI" }, + { NULL, NULL, NULL } }; struct server_context @@ -652,67 +671,218 @@ server_context_free (struct server_context *self) str_map_free (&self->config); } -// --- ? ----------------------------------------------------------------------- +// --- JSON-RPC ---------------------------------------------------------------- + +// TODO: this is where we're actually supposed to do JSON-RPC 2.0 processing + +// There's probably no reason to create an object for this. +// +// We probably just want a handler function that takes a JSON string, parses it, +// and returns back another JSON string. +// +// Then there should be another function that takes a parsed JSON request and +// returns back a JSON reply. This function may get called multiple times if +// the user sends a batch request. + +// --- Requests ---------------------------------------------------------------- + +// TODO: something to read in the headers and decide what to do with the request +// e.g. whether to reject it with a 404, or do JSON-RPC, or ignore it with 200 + +#if 0 +// This doesn't necessarily have to be an object by itself either; we can have +// a function that does/returns something based on the headers + +struct request +{ +}; + +static void +request_init (struct request *self) +{ +} + +static void +request_free (struct request *self) +{ +} +#endif + +// --- Client communication handlers ------------------------------------------- + +struct client +{ + LIST_HEADER (struct client) + + struct server_context *ctx; ///< Server context + + int socket_fd; ///< The TCP socket + write_queue_t write_queue; ///< Write queue + + ev_io read_watcher; ///< The socket can be read from + ev_io write_watcher; ///< The socket can be written to + + struct client_impl *impl; ///< Client behaviour + void *impl_data; ///< Client behaviour data +}; + +struct client_impl +{ + /// Initialize the client as needed + void (*init) (struct client *client); + + /// Do any additional cleanup + void (*destroy) (struct client *client); + + /// Process incoming data; "len == 0" means EOF + bool (*on_data) (struct client *client, const void *data, size_t len); +}; + +static void +client_init (struct client *self) +{ + memset (self, 0, sizeof *self); + write_queue_init (&self->write_queue); +} + +static void +client_free (struct client *self) +{ + write_queue_free (&self->write_queue); +} + +static void +client_write (struct client *client, const void *data, size_t len) +{ + write_req_t *req = xcalloc (1, sizeof *req); + req->data.iov_base = memcpy (xmalloc (len), data, len); + req->data.iov_len = len; + + write_queue_add (&client->write_queue, req); + ev_io_start (EV_DEFAULT_ &client->write_watcher); +} + +// - - FastCGI - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +struct client_fcgi +{ + struct fcgi_parser parser; ///< FastCGI stream parser +}; + +static void +client_fcgi_init (struct client *client) +{ + struct client_fcgi *self = xcalloc (1, sizeof *self); + client->impl_data = self; + + fcgi_parser_init (&self->parser); + // TODO: configure the parser +} + +static void +client_fcgi_destroy (struct client *client) +{ + struct client_fcgi *self = client->impl_data; + client->impl_data = NULL; + + fcgi_parser_free (&self->parser); + free (self); +} + +static bool +client_fcgi_on_data (struct client *client, const void *data, size_t len) +{ + struct client_fcgi *self = client->impl_data; + fcgi_parser_push (&self->parser, data, len); + return true; +} + +static struct client_impl g_client_fcgi = +{ + .init = client_fcgi_init, + .destroy = client_fcgi_destroy, + .on_data = client_fcgi_on_data, +}; + +// - - SCGI - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +struct client_scgi +{ + struct scgi_parser parser; ///< SCGI stream parser +}; + +static void +client_scgi_init (struct client *client) +{ + struct client_scgi *self = xcalloc (1, sizeof *self); + client->impl_data = self; + + scgi_parser_init (&self->parser); + // TODO: configure the parser +} + +static void +client_scgi_destroy (struct client *client) +{ + struct client_scgi *self = client->impl_data; + client->impl_data = NULL; + + scgi_parser_free (&self->parser); + free (self); +} + +static bool +client_scgi_on_data (struct client *client, const void *data, size_t len) +{ + struct client_scgi *self = client->impl_data; + struct error *e = NULL; + if (scgi_parser_push (&self->parser, data, len, &e)) + return true; + + print_debug ("SCGI parser failed: %s", e->message); + error_free (e); + return false; +} + +static struct client_impl g_client_scgi = +{ + .init = client_scgi_init, + .destroy = client_scgi_destroy, + .on_data = client_scgi_on_data, +}; + +// --- Basic server stuff ------------------------------------------------------ + +struct listener +{ + int fd; ///< Listening socket FD + ev_io watcher; ///< New connection available + struct client_impl *impl; ///< Client behaviour +}; static void remove_client (struct server_context *ctx, struct client *client) { + LIST_UNLINK (ctx->clients, client); + ctx->n_clients--; + + // First uninitialize the higher-level implementation + client->impl->destroy (client); + ev_io_stop (EV_DEFAULT_ &client->read_watcher); ev_io_stop (EV_DEFAULT_ &client->write_watcher); xclose (client->socket_fd); - LIST_UNLINK (ctx->clients, client); client_free (client); free (client); } static bool -read_loop (EV_P_ ev_io *watcher, - bool (*cb) (EV_P_ ev_io *, const void *, ssize_t)) +on_client_data (EV_P_ ev_io *watcher, const void *buf, ssize_t n_read) { - char buf[8192]; - while (true) - { - ssize_t n_read = recv (watcher->fd, buf, sizeof buf, 0); - if (n_read < 0) - { - if (errno == EAGAIN) - break; - if (errno == EINTR) - continue; - } - if (n_read <= 0 || !cb (EV_A_ watcher, buf, n_read)) - return false; - } - return true; -} + (void) loop; -static bool -flush_queue (write_queue_t *queue, ev_io *watcher) -{ - struct iovec vec[queue->len], *vec_iter = vec; - for (write_req_t *iter = queue->head; iter; iter = iter->next) - *vec_iter++ = iter->data; - - ssize_t written; -again: - written = writev (watcher->fd, vec, N_ELEMENTS (vec)); - if (written < 0) - { - if (errno == EAGAIN) - goto skip; - if (errno == EINTR) - goto again; - return false; - } - - write_queue_processed (queue, written); - -skip: - if (write_queue_is_empty (queue)) - ev_io_stop (EV_DEFAULT_ watcher); - else - ev_io_start (EV_DEFAULT_ watcher); - return true; + struct client *client = watcher->data; + return client->impl->on_data (client, buf, n_read); } static void @@ -734,12 +904,12 @@ error: } static void -on_fcgi_client_available (EV_P_ ev_io *watcher, int revents) +on_client_available (EV_P_ ev_io *watcher, int revents) { struct server_context *ctx = ev_userdata (loop); + struct listener *listener = watcher->data; (void) revents; - // TODO while (true) { int sock_fd = accept (watcher->fd, NULL, NULL); @@ -759,11 +929,13 @@ on_fcgi_client_available (EV_P_ ev_io *watcher, int revents) break; } + set_blocking (sock_fd, false); + struct client *client = xmalloc (sizeof *client); client_init (client); client->socket_fd = sock_fd; + client->impl = listener->impl; - set_blocking (sock_fd, false); ev_io_init (&client->read_watcher, on_client_ready, sock_fd, EV_READ); ev_io_init (&client->write_watcher, on_client_ready, sock_fd, EV_WRITE); client->read_watcher.data = client; @@ -772,6 +944,9 @@ on_fcgi_client_available (EV_P_ ev_io *watcher, int revents) // We're only interested in reading as the write queue is empty now ev_io_start (EV_A_ &client->read_watcher); + // Initialize the higher-level implementation + client->impl->init (client); + LIST_PREPEND (ctx->clients, client); ctx->n_clients++; } @@ -792,7 +967,7 @@ parse_config (struct server_context *ctx, struct error **e) } static int -listen_finish (struct addrinfo *gai_iter) +listener_finish (struct addrinfo *gai_iter) { int fd = socket (gai_iter->ai_family, gai_iter->ai_socktype, gai_iter->ai_protocol); @@ -832,8 +1007,8 @@ listen_finish (struct addrinfo *gai_iter) } static void -listen_resolve (struct server_context *ctx, const char *host, const char *port, - struct addrinfo *gai_hints, enum listener_type type) +listener_add (struct server_context *ctx, const char *host, const char *port, + struct addrinfo *gai_hints, struct client_impl *impl) { struct addrinfo *gai_result, *gai_iter; int err = getaddrinfo (host, port, gai_hints, &gai_result); @@ -849,24 +1024,15 @@ listen_resolve (struct server_context *ctx, const char *host, const char *port, int fd; for (gai_iter = gai_result; gai_iter; gai_iter = gai_iter->ai_next) { - if ((fd = listen_finish (gai_iter)) == -1) + if ((fd = listener_finish (gai_iter)) == -1) continue; set_blocking (fd, false); struct listener *listener = &ctx->listeners[ctx->n_listeners++]; - switch ((listener->type = type)) - { - case LISTENER_FCGI: - ev_io_init (&listener->watcher, - on_fcgi_client_available, fd, EV_READ); - break; - case LISTENER_SCGI: - ev_io_init (&listener->watcher, - on_scgi_client_available, fd, EV_READ); - break; - } - + ev_io_init (&listener->watcher, on_client_available, fd, EV_READ); ev_io_start (EV_DEFAULT_ &listener->watcher); + listener->watcher.data = listener; + listener->impl = impl; break; } freeaddrinfo (gai_result); @@ -885,10 +1051,8 @@ setup_listen_fds (struct server_context *ctx, struct error **e) gai_hints.ai_socktype = SOCK_STREAM; gai_hints.ai_flags = AI_PASSIVE; - struct str_vector ports_fcgi; - struct str_vector ports_scgi; - str_vector_init (&ports_fcgi); - str_vector_init (&ports_scgi); + struct str_vector ports_fcgi; str_vector_init (&ports_fcgi); + struct str_vector ports_scgi; str_vector_init (&ports_scgi); if (port_fcgi) split_str_ignore_empty (port_fcgi, ',', &ports_fcgi); @@ -899,11 +1063,11 @@ setup_listen_fds (struct server_context *ctx, struct error **e) ctx->listeners = xcalloc (n_ports, sizeof *ctx->listeners); for (size_t i = 0; i < ports_fcgi.len; i++) - listen_resolve (ctx, bind_host, ports_fcgi.vector[i], - &gai_hints, LISTENER_FCGI); + listener_add (ctx, bind_host, ports_fcgi.vector[i], + &gai_hints, &g_client_fcgi); for (size_t i = 0; i < ports_scgi.len; i++) - listen_resolve (ctx, bind_host, ports_scgi.vector[i], - &gai_hints, LISTENER_SCGI); + listener_add (ctx, bind_host, ports_scgi.vector[i], + &gai_hints, &g_client_scgi); str_vector_free (&ports_fcgi); str_vector_free (&ports_scgi); From 0b0d64124b391d03ffe2b535b5b23d4ae3655fa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Thu, 5 Mar 2015 08:47:20 +0100 Subject: [PATCH 03/79] Steady progress --- demo-json-rpc-server.c | 216 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 206 insertions(+), 10 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index 8645ae3..de8a542 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -427,14 +427,25 @@ fcgi_nv_parser_push (struct fcgi_nv_parser *self, void *data, size_t len) // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - // TODO + +enum fcgi_request_state +{ + FCGI_REQUEST_PARAMS, ///< Reading headers + FCGI_REQUEST_STDIN ///< Reading input +}; + struct fcgi_request { struct fcgi_muxer *muxer; ///< The parent muxer uint16_t request_id; ///< The ID of this request + enum fcgi_request_state state; ///< Parsing state + struct str_map headers; ///< Headers + struct fcgi_nv_parser hdr_parser; ///< Header parser }; // TODO + struct fcgi_muxer { struct fcgi_parser parser; ///< FastCGI message parser @@ -492,6 +503,14 @@ struct scgi_parser size_t headers_len; ///< Length of the netstring contents struct str name; ///< Header name so far struct str value; ///< Header value so far + + /// Finished parsing request headers + void (*on_headers_read) (void *user_data); + + /// Content available; len == 0 means end of file + void (*on_content) (void *user_data, const void *data, size_t len); + + void *user_data; ///< User data passed to callbacks }; static void @@ -525,7 +544,8 @@ scgi_parser_push (struct scgi_parser *self, return false; } - // TODO: a "on_eof" callback? + // Indicate end of file + self->on_content (self->user_data, NULL, 0); return true; } @@ -621,10 +641,9 @@ scgi_parser_push (struct scgi_parser *self, break; } case SCGI_READING_CONTENT: - // TODO: a "on_content" callback? + self->on_content (self->user_data, self->input.str, self->input.len); + str_remove_slice (&self->input, 0, self->input.len); return true; - - break; } } @@ -684,29 +703,158 @@ server_context_free (struct server_context *self) // returns back a JSON reply. This function may get called multiple times if // the user sends a batch request. +static bool +try_advance (const char **p, const char *text) +{ + size_t len = strlen (text); + if (strncmp (*p, text, len)) + return false; + + *p += len; + return true; +} + +static bool +validate_json_rpc_content_type (const char *type) +{ + const char *content_types[] = + { + "application/json-rpc", // obsolete + "application/json" + }; + const char *tails[] = + { + "; charset=utf-8", + "; charset=UTF-8", + "" + }; + + bool found = false; + for (size_t i = 0; i < N_ELEMENTS (content_types); i++) + if ((found = try_advance (&type, content_types[i]))) + break; + if (!found) + return false; + + for (size_t i = 0; i < N_ELEMENTS (tails); i++) + if ((found = try_advance (&type, tails[i]))) + break; + if (!found) + return false; + + return !*type; +} + // --- Requests ---------------------------------------------------------------- // TODO: something to read in the headers and decide what to do with the request // e.g. whether to reject it with a 404, or do JSON-RPC, or ignore it with 200 -#if 0 -// This doesn't necessarily have to be an object by itself either; we can have -// a function that does/returns something based on the headers - struct request { + // TODO *ctx + + void *user_data; ///< User data argument for callbacks + + /// Callback to write some CGI response data to the output + void (*write_cb) (void *user_data, const void *data, size_t len); + + /// Callback to close the connection + void (*close_cb) (void *user_data); + + struct request_handler *handler; ///< Current request handler + void *handler_data; ///< User data for the handler +}; + +struct request_handler +{ + /// Install ourselves as the handler for the request if applicable + bool (*try_handle) (struct request *request, struct str_map *headers); + + /// Handle incoming data + void (*push_cb) (struct request *request, const void *data, size_t len); + + /// Destroy the handler + void (*destroy_cb) (struct request *request); }; static void request_init (struct request *self) { + memset (self, 0, sizeof *self); } static void request_free (struct request *self) { + // TODO: destroy the handler? } -#endif + +static void +request_start (struct request *self, struct str_map *headers) +{ + bool handled = false; + // TODO: try request handlers registered in self->ctx + if (handled) + return; + + // Unable to serve the request + struct str response; + str_init (&response); + str_append (&response, "404 Not Found\r\n\r\n"); + self->write_cb (self->user_data, response.str, response.len); + str_free (&response); + + // XXX: how will the clients behave when this happens? + self->close_cb (self->user_data); +} + +static void +request_push (struct request *self, const void *data, size_t len) +{ + if (soft_assert (self->handler)) + self->handler->push_cb (self, data, len); +} + +// --- Requests handlers ------------------------------------------------------- + +static bool +request_handler_json_rpc_try_handle + (struct request *request, struct str_map *headers) +{ + const char *content_type = str_map_find (headers, "CONTENT_TYPE"); + const char *method = str_map_find (headers, "REQUEST_METHOD"); + + if (strcmp (method, "POST") + || !validate_json_rpc_content_type (content_type)) + return false; + + // TODO: install the handler, perhaps construct an object + return true; +} + +static void +request_handler_json_rpc_push + (struct request *request, const void *data, size_t len) +{ + // TODO: append to a buffer + // TODO: len == 0: process the request +} + +static void +request_handler_json_rpc_destroy (struct request *request) +{ + // TODO +} + +struct request_handler g_request_handler_json_rpc = +{ + .try_handle = request_handler_json_rpc_try_handle, + .push_cb = request_handler_json_rpc_push, + .destroy_cb = request_handler_json_rpc_destroy, +}; + +// TODO: another request handler to respond to all GETs with a message // --- Client communication handlers ------------------------------------------- @@ -809,16 +957,59 @@ static struct client_impl g_client_fcgi = struct client_scgi { struct scgi_parser parser; ///< SCGI stream parser + struct request request; ///< Request }; +static void +client_scgi_write (void *user_data, const void *data, size_t len) +{ + struct client *client = user_data; + client_write (client, data, len); +} + +static void +client_scgi_close (void *user_data) +{ + struct client *client = user_data; + struct client_scgi *self = client->impl_data; + + // TODO +} + +static void +client_scgi_on_headers_read (void *user_data) +{ + struct client *client = user_data; + struct client_scgi *self = client->impl_data; + + request_start (&self->request, &self->parser.headers); +} + +static void +client_scgi_on_content (void *user_data, const void *data, size_t len) +{ + struct client *client = user_data; + struct client_scgi *self = client->impl_data; + + // XXX: make sure this is understood as EOF + request_push (&self->request, data, len); +} + static void client_scgi_init (struct client *client) { struct client_scgi *self = xcalloc (1, sizeof *self); client->impl_data = self; + request_init (&self->request); + self->request.write_cb = client_scgi_write; + self->request.close_cb = client_scgi_close; + self->request.user_data = client; + scgi_parser_init (&self->parser); - // TODO: configure the parser + self->parser.on_headers_read = client_scgi_on_headers_read; + self->parser.on_content = client_scgi_on_content; + self->parser.user_data = client; } static void @@ -827,6 +1018,9 @@ client_scgi_destroy (struct client *client) struct client_scgi *self = client->impl_data; client->impl_data = NULL; + // TODO: do something more to abort the request? + request_free (&self->request); + scgi_parser_free (&self->parser); free (self); } @@ -891,6 +1085,8 @@ on_client_ready (EV_P_ ev_io *watcher, int revents) struct server_context *ctx = ev_userdata (loop); struct client *client = watcher->data; + // FIXME: don't close the connection on EOF; we need to be able to keep + // the connection open and respond in an asynchronous manner if (revents & EV_READ) if (!read_loop (EV_A_ watcher, on_client_data)) goto error; From 2733ead30f01c7391a66c492261d9bfb247a8823 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Fri, 6 Mar 2015 19:49:33 +0100 Subject: [PATCH 04/79] Figuring out how to close the connection --- demo-json-rpc-server.c | 204 ++++++++++++++++++++++------------------- 1 file changed, 112 insertions(+), 92 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index de8a542..09f6002 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -68,30 +68,6 @@ msg_unpacker_u32 (struct msg_unpacker *self, uint32_t *value) // --- libev helpers ----------------------------------------------------------- -static bool -read_loop (EV_P_ ev_io *watcher, - bool (*cb) (EV_P_ ev_io *, const void *, ssize_t)) -{ - char buf[8192]; - while (true) - { - ssize_t n_read = recv (watcher->fd, buf, sizeof buf, 0); - if (n_read < 0) - { - if (errno == EAGAIN) - break; - if (errno == EINTR) - continue; - } - // The callback is called on EOF as well - if (n_read < 0 || !cb (EV_A_ watcher, buf, n_read)) - return false; - if (!n_read) - return false; - } - return true; -} - static bool flush_queue (write_queue_t *queue, ev_io *watcher) { @@ -504,11 +480,13 @@ struct scgi_parser struct str name; ///< Header name so far struct str value; ///< Header value so far - /// Finished parsing request headers - void (*on_headers_read) (void *user_data); + /// Finished parsing request headers. + /// Return false to abort further processing of input. + bool (*on_headers_read) (void *user_data); - /// Content available; len == 0 means end of file - void (*on_content) (void *user_data, const void *data, size_t len); + /// Content available; len == 0 means end of file. + /// Return false to abort further processing of input. + bool (*on_content) (void *user_data, const void *data, size_t len); void *user_data; ///< User data passed to callbacks }; @@ -545,15 +523,15 @@ scgi_parser_push (struct scgi_parser *self, } // Indicate end of file - self->on_content (self->user_data, NULL, 0); - return true; + return self->on_content (self->user_data, NULL, 0); } // Notice that this madness is significantly harder to parse than FastCGI; // this procedure could also be optimized significantly str_append_data (&self->input, data, len); - while (true) + bool keep_running = true; + while (keep_running) switch (self->state) { case SCGI_READING_NETSTRING_LENGTH: @@ -601,7 +579,7 @@ scgi_parser_push (struct scgi_parser *self, return false; } self->state = SCGI_READING_CONTENT; - // TODO: a "on_headers_read" callback? + keep_running = self->on_headers_read (self->user_data); } else if (c != '\0') str_append_c (&self->name, c); @@ -641,10 +619,12 @@ scgi_parser_push (struct scgi_parser *self, break; } case SCGI_READING_CONTENT: - self->on_content (self->user_data, self->input.str, self->input.len); + keep_running = self->on_content + (self->user_data, self->input.str, self->input.len); str_remove_slice (&self->input, 0, self->input.len); - return true; + return keep_running; } + return false; } // --- Server ------------------------------------------------------------------ @@ -703,6 +683,9 @@ server_context_free (struct server_context *self) // returns back a JSON reply. This function may get called multiple times if // the user sends a batch request. +// TODO: a function that queues up a ping over IRC: this has to be owned by the +// server context as a background job that removes itself upon completion. + static bool try_advance (const char **p, const char *text) { @@ -747,9 +730,6 @@ validate_json_rpc_content_type (const char *type) // --- Requests ---------------------------------------------------------------- -// TODO: something to read in the headers and decide what to do with the request -// e.g. whether to reject it with a 404, or do JSON-RPC, or ignore it with 200 - struct request { // TODO *ctx @@ -759,7 +739,8 @@ struct request /// Callback to write some CGI response data to the output void (*write_cb) (void *user_data, const void *data, size_t len); - /// Callback to close the connection + /// Callback to close the connection. + /// CALLING THIS MAY CAUSE THE REQUEST TO BE DESTROYED. void (*close_cb) (void *user_data); struct request_handler *handler; ///< Current request handler @@ -771,8 +752,9 @@ struct request_handler /// Install ourselves as the handler for the request if applicable bool (*try_handle) (struct request *request, struct str_map *headers); - /// Handle incoming data - void (*push_cb) (struct request *request, const void *data, size_t len); + /// Handle incoming data. + /// Return false if further processing should be stopped. + bool (*push_cb) (struct request *request, const void *data, size_t len); /// Destroy the handler void (*destroy_cb) (struct request *request); @@ -787,16 +769,27 @@ request_init (struct request *self) static void request_free (struct request *self) { - // TODO: destroy the handler? + if (self->handler) + self->handler->destroy_cb (self); } +/// This function is only intended to be run from asynchronous event handlers +/// such as timers, not as a direct result of starting the request or receiving +/// request data. CALLING THIS MAY CAUSE THE REQUEST TO BE DESTROYED. static void +request_finish (struct request *self) +{ + self->close_cb (self->user_data); +} + +static bool request_start (struct request *self, struct str_map *headers) { bool handled = false; // TODO: try request handlers registered in self->ctx if (handled) - return; + // TODO: can also be false + return true; // Unable to serve the request struct str response; @@ -804,16 +797,17 @@ request_start (struct request *self, struct str_map *headers) str_append (&response, "404 Not Found\r\n\r\n"); self->write_cb (self->user_data, response.str, response.len); str_free (&response); - - // XXX: how will the clients behave when this happens? - self->close_cb (self->user_data); + return false; } -static void +static bool request_push (struct request *self, const void *data, size_t len) { if (soft_assert (self->handler)) - self->handler->push_cb (self, data, len); + return self->handler->push_cb (self, data, len); + + // No handler, nothing to do with any data + return false; } // --- Requests handlers ------------------------------------------------------- @@ -833,12 +827,13 @@ request_handler_json_rpc_try_handle return true; } -static void +static bool request_handler_json_rpc_push (struct request *request, const void *data, size_t len) { // TODO: append to a buffer // TODO: len == 0: process the request + return true; } static void @@ -883,7 +878,7 @@ struct client_impl void (*destroy) (struct client *client); /// Process incoming data; "len == 0" means EOF - bool (*on_data) (struct client *client, const void *data, size_t len); + bool (*push) (struct client *client, const void *data, size_t len); }; static void @@ -910,6 +905,24 @@ client_write (struct client *client, const void *data, size_t len) ev_io_start (EV_DEFAULT_ &client->write_watcher); } +static void +client_remove (struct client *client) +{ + struct server_context *ctx = client->ctx; + + LIST_UNLINK (ctx->clients, client); + ctx->n_clients--; + + // First uninitialize the higher-level implementation + client->impl->destroy (client); + + ev_io_stop (EV_DEFAULT_ &client->read_watcher); + ev_io_stop (EV_DEFAULT_ &client->write_watcher); + xclose (client->socket_fd); + client_free (client); + free (client); +} + // - - FastCGI - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - struct client_fcgi @@ -938,7 +951,7 @@ client_fcgi_destroy (struct client *client) } static bool -client_fcgi_on_data (struct client *client, const void *data, size_t len) +client_fcgi_push (struct client *client, const void *data, size_t len) { struct client_fcgi *self = client->impl_data; fcgi_parser_push (&self->parser, data, len); @@ -949,7 +962,7 @@ static struct client_impl g_client_fcgi = { .init = client_fcgi_init, .destroy = client_fcgi_destroy, - .on_data = client_fcgi_on_data, + .push = client_fcgi_push, }; // - - SCGI - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -957,7 +970,7 @@ static struct client_impl g_client_fcgi = struct client_scgi { struct scgi_parser parser; ///< SCGI stream parser - struct request request; ///< Request + struct request request; ///< Request (only one per connection) }; static void @@ -970,29 +983,29 @@ client_scgi_write (void *user_data, const void *data, size_t len) static void client_scgi_close (void *user_data) { + // XXX: this rather really means "close me [the request]" struct client *client = user_data; - struct client_scgi *self = client->impl_data; - - // TODO + client_remove (client); } -static void +static bool client_scgi_on_headers_read (void *user_data) { struct client *client = user_data; struct client_scgi *self = client->impl_data; - - request_start (&self->request, &self->parser.headers); + return request_start (&self->request, &self->parser.headers); } -static void +static bool client_scgi_on_content (void *user_data, const void *data, size_t len) { struct client *client = user_data; struct client_scgi *self = client->impl_data; - // XXX: make sure this is understood as EOF - request_push (&self->request, data, len); + // XXX: do we have to count CONTENT_LENGTH and supply our own EOF? + // If we do produce our own EOF, we should probably make sure we don't + // send it twice in a row. + return request_push (&self->request, data, len); } static void @@ -1018,23 +1031,24 @@ client_scgi_destroy (struct client *client) struct client_scgi *self = client->impl_data; client->impl_data = NULL; - // TODO: do something more to abort the request? request_free (&self->request); - scgi_parser_free (&self->parser); free (self); } static bool -client_scgi_on_data (struct client *client, const void *data, size_t len) +client_scgi_push (struct client *client, const void *data, size_t len) { struct client_scgi *self = client->impl_data; struct error *e = NULL; if (scgi_parser_push (&self->parser, data, len, &e)) return true; - print_debug ("SCGI parser failed: %s", e->message); - error_free (e); + if (e != NULL) + { + print_debug ("SCGI parser failed: %s", e->message); + error_free (e); + } return false; } @@ -1042,7 +1056,7 @@ static struct client_impl g_client_scgi = { .init = client_scgi_init, .destroy = client_scgi_destroy, - .on_data = client_scgi_on_data, + .push = client_scgi_push, }; // --- Basic server stuff ------------------------------------------------------ @@ -1054,41 +1068,45 @@ struct listener struct client_impl *impl; ///< Client behaviour }; -static void -remove_client (struct server_context *ctx, struct client *client) -{ - LIST_UNLINK (ctx->clients, client); - ctx->n_clients--; - - // First uninitialize the higher-level implementation - client->impl->destroy (client); - - ev_io_stop (EV_DEFAULT_ &client->read_watcher); - ev_io_stop (EV_DEFAULT_ &client->write_watcher); - xclose (client->socket_fd); - client_free (client); - free (client); -} - static bool -on_client_data (EV_P_ ev_io *watcher, const void *buf, ssize_t n_read) +client_read_loop (EV_P_ struct client *client, ev_io *watcher) { - (void) loop; + char buf[8192]; + while (true) + { + ssize_t n_read = recv (watcher->fd, buf, sizeof buf, 0); + if (n_read < 0) + { + if (errno == EAGAIN) + break; + if (errno == EINTR) + continue; - struct client *client = watcher->data; - return client->impl->on_data (client, buf, n_read); + return false; + } + + if (!client->impl->push (client, buf, n_read)) + return false; + + if (!n_read) + { + // Don't receive the EOF condition repeatedly + ev_io_stop (EV_A_ watcher); + + // We can probably still write, so let's just return + return true; + } + } + return true; } static void on_client_ready (EV_P_ ev_io *watcher, int revents) { - struct server_context *ctx = ev_userdata (loop); struct client *client = watcher->data; - // FIXME: don't close the connection on EOF; we need to be able to keep - // the connection open and respond in an asynchronous manner if (revents & EV_READ) - if (!read_loop (EV_A_ watcher, on_client_data)) + if (!client_read_loop (EV_A_ client, watcher)) goto error; if (revents & EV_WRITE) if (!flush_queue (&client->write_queue, watcher)) @@ -1096,7 +1114,9 @@ on_client_ready (EV_P_ ev_io *watcher, int revents) return; error: - remove_client (ctx, client); + // The callback also could have just told us to stop reading, + // this is not necessarily an error condition + client_remove (client); } static void From 9e0c9dd6d8bf65ab76c2b54f5038ff07c0d8d6d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Sun, 8 Mar 2015 05:39:57 +0100 Subject: [PATCH 05/79] Steady progress Still trying to figure out FastCGI. At least I've finally implemented the JSON-RPC handler. --- demo-json-rpc-server.c | 684 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 650 insertions(+), 34 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index 09f6002..6be6b3e 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -66,6 +66,45 @@ msg_unpacker_u32 (struct msg_unpacker *self, uint32_t *value) #undef UNPACKER_INT_BEGIN +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +// "msg_writer" should be rewritten on top of this + +static void +str_pack_u8 (struct str *self, uint8_t x) +{ + str_append_data (self, &x, 1); +} + +static void +str_pack_u16 (struct str *self, uint64_t x) +{ + uint8_t tmp[2] = { x >> 8, x }; + str_append_data (self, tmp, sizeof tmp); +} + +static void +str_pack_u32 (struct str *self, uint32_t x) +{ + uint32_t u = x; + uint8_t tmp[4] = { u >> 24, u >> 16, u >> 8, u }; + str_append_data (self, tmp, sizeof tmp); +} + +static void +str_pack_i32 (struct str *self, int32_t x) +{ + str_pack_u32 (self, (uint32_t) x); +} + +static void +str_pack_u64 (struct str *self, uint64_t x) +{ + uint8_t tmp[8] = + { x >> 56, x >> 48, x >> 40, x >> 32, x >> 24, x >> 16, x >> 8, x }; + str_append_data (self, tmp, sizeof tmp); +} + // --- libev helpers ----------------------------------------------------------- static bool @@ -312,7 +351,7 @@ fcgi_nv_parser_free (struct fcgi_nv_parser *self) } static void -fcgi_nv_parser_push (struct fcgi_nv_parser *self, void *data, size_t len) +fcgi_nv_parser_push (struct fcgi_nv_parser *self, const void *data, size_t len) { // This could be optimized significantly; I'm not even trying str_append_data (&self->input, data, len); @@ -402,7 +441,38 @@ fcgi_nv_parser_push (struct fcgi_nv_parser *self, void *data, size_t len) // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -// TODO +static void +fcgi_nv_convert_len (size_t len, struct str *output) +{ + if (len < 0x80) + str_pack_u8 (output, len); + else + { + len |= (uint32_t) 1 << 31; + str_pack_u32 (output, len); + } +} + +static void +fcgi_nv_convert (struct str_map *map, struct str *output) +{ + struct str_map_iter iter; + str_map_iter_init (&iter, map); + while (str_map_iter_next (&iter)) + { + const char *name = iter.link->key; + const char *value = iter.link->data; + size_t name_len = iter.link->key_length; + size_t value_len = strlen (value); + + fcgi_nv_convert_len (name_len, output); + fcgi_nv_convert_len (value_len, output); + str_append_data (output, name, name_len); + str_append_data (output, value, value_len); + } +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - enum fcgi_request_state { @@ -413,14 +483,17 @@ enum fcgi_request_state struct fcgi_request { struct fcgi_muxer *muxer; ///< The parent muxer - uint16_t request_id; ///< The ID of this request + uint8_t flags; ///< Request flags + enum fcgi_request_state state; ///< Parsing state struct str_map headers; ///< Headers struct fcgi_nv_parser hdr_parser; ///< Header parser -}; -// TODO + struct str output_buffer; ///< Output buffer + + void *handler_data; ///< Handler data +}; struct fcgi_muxer { @@ -429,14 +502,302 @@ struct fcgi_muxer /// Requests assigned to request IDs // TODO: allocate this dynamically struct fcgi_request *requests[1 << 16]; + + void (*write_cb) (void *user_data, const void *data, size_t len); + void (*close_cb) (void *user_data); + + void *(*request_start_cb) (void *user_data, struct fcgi_request *request); + void (*request_push_cb) (void *handler_data, const void *data, size_t len); + void (*request_destroy_cb) (void *handler_data); + + void *user_data; ///< User data for callbacks }; +static void +fcgi_muxer_send (struct fcgi_muxer *self, + enum fcgi_type type, uint16_t request_id, const void *data, size_t len) +{ + hard_assert (len <= UINT16_MAX); + + struct str message; + str_init (&message); + + str_pack_u8 (&message, FCGI_VERSION_1); + str_pack_u8 (&message, type); + str_pack_u16 (&message, request_id); + str_pack_u16 (&message, len); // content length + str_pack_u8 (&message, 0); // padding length + + str_append_data (&message, data, len); + + // XXX: we should probably have another write_cb that assumes ownership + self->write_cb (self->user_data, message.str, message.len); + str_free (&message); +} + +static void +fcgi_muxer_send_end_request (struct fcgi_muxer *self, uint16_t request_id, + uint32_t app_status, enum fcgi_protocol_status protocol_status) +{ + uint8_t content[8] = { app_status >> 24, app_status >> 16, + app_status << 8, app_status, protocol_status }; + fcgi_muxer_send (self, FCGI_END_REQUEST, request_id, + content, sizeof content); +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +static void +fcgi_request_init (struct fcgi_request *self) +{ + memset (self, 0, sizeof *self); + + str_map_init (&self->headers); + self->headers.free = free; + + fcgi_nv_parser_init (&self->hdr_parser); + self->hdr_parser.output = &self->headers; +} + +static void +fcgi_request_free (struct fcgi_request *self) +{ + str_map_free (&self->headers); + fcgi_nv_parser_free (&self->hdr_parser); +} + +static void +fcgi_request_push_params + (struct fcgi_request *self, const void *data, size_t len) +{ + if (self->state != FCGI_REQUEST_PARAMS) + { + // TODO: probably reject the request + return; + } + + if (len) + fcgi_nv_parser_push (&self->hdr_parser, data, len); + else + { + // TODO: probably check the state of the header parser + // TODO: request_start() can return false, end the request here? + self->handler_data = self->muxer->request_start_cb + (self->muxer->user_data, self); + self->state = FCGI_REQUEST_STDIN; + } +} + +static void +fcgi_request_push_stdin + (struct fcgi_request *self, const void *data, size_t len) +{ + if (self->state != FCGI_REQUEST_STDIN) + { + // TODO: probably reject the request + return; + } + + self->muxer->request_push_cb (self->handler_data, data, len); +} + +static void +fcgi_request_flush (struct fcgi_request *self) +{ + if (!self->output_buffer.len) + return; + + fcgi_muxer_send (self->muxer, FCGI_STDOUT, self->request_id, + self->output_buffer.str, self->output_buffer.len); + str_reset (&self->output_buffer); +} + +static void +fcgi_request_write (struct fcgi_request *self, const void *data, size_t len) +{ + // We're buffering the output and splitting it into messages + bool need_flush = true; + while (len) + { + size_t to_write = UINT16_MAX - self->output_buffer.len; + if (to_write > len) + { + to_write = len; + need_flush = false; + } + + str_append_data (&self->output_buffer, data, to_write); + data = (uint8_t *) data + to_write; + len -= to_write; + + if (need_flush) + fcgi_request_flush (self); + } +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +typedef void (*fcgi_muxer_handler_fn) + (struct fcgi_muxer *, const struct fcgi_parser *); + +static void +fcgi_muxer_on_get_values + (struct fcgi_muxer *self, const struct fcgi_parser *parser) +{ + struct str_map values; str_map_init (&values); values.free = free; + struct str_map response; str_map_init (&response); response.free = free; + + struct fcgi_nv_parser nv_parser; + fcgi_nv_parser_init (&nv_parser); + nv_parser.output = &values; + + fcgi_nv_parser_push (&nv_parser, parser->content.str, parser->content.len); + + struct str_map_iter iter; + str_map_iter_init (&iter, &values); + while (str_map_iter_next (&iter)) + { + const char *key = iter.link->key; + + // TODO: if (!strcmp (key, FCGI_MAX_CONNS)) + // TODO: if (!strcmp (key, FCGI_MAX_REQS)) + + if (!strcmp (key, FCGI_MPXS_CONNS)) + str_map_set (&response, key, xstrdup ("1")); + } + + struct str content; + str_init (&content); + fcgi_nv_convert (&response, &content); + fcgi_muxer_send (self, FCGI_GET_VALUES_RESULT, parser->request_id, + content.str, content.len); + str_free (&content); + + str_map_free (&values); + str_map_free (&response); +} + +static void +fcgi_muxer_on_begin_request + (struct fcgi_muxer *self, const struct fcgi_parser *parser) +{ + struct msg_unpacker unpacker; + msg_unpacker_init (&unpacker, parser->content.str, parser->content.len); + + uint16_t role; + uint8_t flags; + bool success = true; + success &= msg_unpacker_u16 (&unpacker, &role); + success &= msg_unpacker_u8 (&unpacker, &flags); + // Ignoring 5 reserved bytes + + if (!success) + { + print_debug ("FastCGI: ignoring invalid %s message", + STRINGIFY (FCGI_BEGIN_REQUEST)); + return; + } + + if (role != FCGI_RESPONDER) + { + fcgi_muxer_send_end_request (self, + parser->request_id, 0, FCGI_UNKNOWN_ROLE); + return; + } + + struct fcgi_request *request = self->requests[parser->request_id]; + if (request) + { + // TODO: fail + return; + } + + request = xcalloc (1, sizeof *request); + fcgi_request_init (request); + request->muxer = self; + request->request_id = parser->request_id; + request->flags = flags; + + self->requests[parser->request_id] = request; +} + +static void +fcgi_muxer_on_abort_request + (struct fcgi_muxer *self, const struct fcgi_parser *parser) +{ + struct fcgi_request *request = self->requests[parser->request_id]; + if (!request) + { + print_debug ("FastCGI: received %s for an unknown request", + STRINGIFY (FCGI_ABORT_REQUEST)); + return; + } + + // TODO: abort the request +} + +static void +fcgi_muxer_on_params (struct fcgi_muxer *self, const struct fcgi_parser *parser) +{ + struct fcgi_request *request = self->requests[parser->request_id]; + if (!request) + { + print_debug ("FastCGI: received %s for an unknown request", + STRINGIFY (FCGI_PARAMS)); + return; + } + + fcgi_request_push_params (request, + parser->content.str, parser->content.len); +} + +static void +fcgi_muxer_on_stdin (struct fcgi_muxer *self, const struct fcgi_parser *parser) +{ + struct fcgi_request *request = self->requests[parser->request_id]; + if (!request) + { + print_debug ("FastCGI: received %s for an unknown request", + STRINGIFY (FCGI_STDIN)); + return; + } + + fcgi_request_push_stdin (request, + parser->content.str, parser->content.len); +} + static void fcgi_muxer_on_message (const struct fcgi_parser *parser, void *user_data) { struct fcgi_muxer *self = user_data; - // TODO + if (parser->version != FCGI_VERSION_1) + { + print_debug ("FastCGI: unsupported version %d", parser->version); + // TODO: also return false to stop processing on protocol error? + return; + } + + static const fcgi_muxer_handler_fn handlers[] = + { + [FCGI_GET_VALUES] = fcgi_muxer_on_get_values, + [FCGI_BEGIN_REQUEST] = fcgi_muxer_on_begin_request, + [FCGI_ABORT_REQUEST] = fcgi_muxer_on_abort_request, + [FCGI_PARAMS] = fcgi_muxer_on_params, + [FCGI_STDIN] = fcgi_muxer_on_stdin, + }; + + fcgi_muxer_handler_fn handler; + if (parser->type >= N_ELEMENTS (handlers) + || !(handler = handlers[parser->type])) + { + uint8_t content[8] = { parser->type }; + fcgi_muxer_send (self, FCGI_UNKNOWN_TYPE, parser->request_id, + content, sizeof content); + return; + } + + handler (self, parser); } static void @@ -649,6 +1010,7 @@ struct server_context struct client *clients; ///< Clients unsigned n_clients; ///< Current number of connections + struct request_handler *handlers; ///< Request handlers struct str_map config; ///< Server configuration }; @@ -672,19 +1034,58 @@ server_context_free (struct server_context *self) // --- JSON-RPC ---------------------------------------------------------------- -// TODO: this is where we're actually supposed to do JSON-RPC 2.0 processing +#define JSON_RPC_ERROR_TABLE(XX) \ + XX (-32700, PARSE_ERROR, "Parse error") \ + XX (-32600, INVALID_REQUEST, "Invalid Request") \ + XX (-32601, METHOD_NOT_FOUND, "Method not found") \ + XX (-32602, INVALID_PARAMS, "Invalid params") \ + XX (-32603, INTERNAL_ERROR, "Internal error") -// There's probably no reason to create an object for this. -// -// We probably just want a handler function that takes a JSON string, parses it, -// and returns back another JSON string. -// -// Then there should be another function that takes a parsed JSON request and -// returns back a JSON reply. This function may get called multiple times if -// the user sends a batch request. +enum json_rpc_error +{ +#define XX(code, name, message) JSON_RPC_ERROR_ ## name, + JSON_RPC_ERROR_TABLE (XX) +#undef XX + JSON_RPC_ERROR_COUNT +}; -// TODO: a function that queues up a ping over IRC: this has to be owned by the -// server context as a background job that removes itself upon completion. +static json_t * +json_rpc_error (enum json_rpc_error id, json_t *data) +{ +#define XX(code, name, message) { code, message }, + static const struct json_rpc_error + { + int code; + const char *message; + } + errors[JSON_RPC_ERROR_COUNT] = + { + JSON_RPC_ERROR_TABLE (XX) + }; +#undef XX + + json_t *error = json_object (); + json_object_set_new (error, "code", json_integer (errors[id].code)); + json_object_set_new (error, "message", json_string (errors[id].message)); + + if (data) + json_object_set_new (error, "data", data); + + return error; +} + +static json_t * +json_rpc_response (json_t *id, json_t *result, json_t *error) +{ + json_t *x = json_object (); + json_object_set_new (x, "jsonrpc", json_string ("2.0")); + json_object_set_new (x, "id", id ? id : json_null ()); + if (result) json_object_set_new (x, "result", result); + if (error) json_object_set_new (x, "error", error); + return x; +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - static bool try_advance (const char **p, const char *text) @@ -728,11 +1129,128 @@ validate_json_rpc_content_type (const char *type) return !*type; } +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +// TODO: a method that queues up a ping over IRC: this has to be owned by the +// server context as a background job that removes itself upon completion. + +static json_t * +json_rpc_ping (struct server_context *ctx, json_t *params) +{ + (void) ctx; + (void) params; + + return json_rpc_response (NULL, json_string ("pong"), NULL); +} + +static json_t * +process_json_rpc_request (struct server_context *ctx, json_t *request) +{ + // TODO: takes a parsed JSON request and returns back a JSON reply. + // This function may get called multiple times for batch requests. + + if (!json_is_object (request)) + return json_rpc_response (NULL, NULL, + json_rpc_error (JSON_RPC_ERROR_INVALID_REQUEST, NULL)); + + json_t *v = json_object_get (request, "jsonrpc"); + json_t *m = json_object_get (request, "method"); + json_t *params = json_object_get (request, "params"); + json_t *id = json_object_get (request, "id"); + + const char *version; + const char *method; + + bool ok = true; + ok &= v && (version = json_string_value (v)) && !strcmp (version, "2.0"); + ok &= m && (method = json_string_value (m)); + ok &= !params || json_is_array (params) || json_is_object (params); + ok &= !id || json_is_null (id) || + json_is_string (id) || json_is_number (id); + if (!ok) + return json_rpc_response (id, NULL, + json_rpc_error (JSON_RPC_ERROR_INVALID_REQUEST, NULL)); + + // TODO: add a more extensible mechanism + json_t *response = NULL; + if (!strcmp (method, "ping")) + response = json_rpc_ping (ctx, params); + else + return json_rpc_response (id, NULL, + json_rpc_error (JSON_RPC_ERROR_METHOD_NOT_FOUND, NULL)); + + if (id) + return response; + + // Notifications don't get responses + // TODO: separate notifications from non-notifications? + json_decref (response); + return NULL; +} + +static void +flush_json (json_t *json, struct str *output) +{ + char *utf8 = json_dumps (json, JSON_ENCODE_ANY); + str_append (output, utf8); + free (utf8); + json_decref (json); +} + +static void +process_json_rpc (struct server_context *ctx, + const void *data, size_t len, struct str *output) +{ + + json_error_t e; + json_t *request; + if (!(request = json_loadb (data, len, JSON_DECODE_ANY, &e))) + { + flush_json (json_rpc_response (NULL, NULL, + json_rpc_error (JSON_RPC_ERROR_PARSE_ERROR, NULL)), + output); + return; + } + + if (json_is_array (request)) + { + if (!json_array_size (request)) + { + flush_json (json_rpc_response (NULL, NULL, + json_rpc_error (JSON_RPC_ERROR_INVALID_REQUEST, NULL)), + output); + return; + } + + json_t *response = json_array (); + json_t *iter; + size_t i; + + json_array_foreach (request, i, iter) + { + json_t *result = process_json_rpc_request (ctx, iter); + if (result) + json_array_append_new (response, result); + } + + if (json_array_size (response)) + flush_json (response, output); + else + json_decref (response); + } + else + { + json_t *result = process_json_rpc_request (ctx, request); + if (result) + flush_json (result, output); + } +} + // --- Requests ---------------------------------------------------------------- struct request { - // TODO *ctx + struct server_context *ctx; ///< Server context void *user_data; ///< User data argument for callbacks @@ -749,6 +1267,8 @@ struct request struct request_handler { + LIST_HEADER (struct request_handler) + /// Install ourselves as the handler for the request if applicable bool (*try_handle) (struct request *request, struct str_map *headers); @@ -785,11 +1305,17 @@ request_finish (struct request *self) static bool request_start (struct request *self, struct str_map *headers) { - bool handled = false; - // TODO: try request handlers registered in self->ctx - if (handled) - // TODO: can also be false - return true; + LIST_FOR_EACH (struct request_handler, handler, self->ctx->handlers) + if (handler->try_handle (self, headers)) + { + // XXX: maybe we should isolate the handlers a bit more + self->handler = handler; + + // TODO: we should also allow the "try_handle" function to + // return that it has already finished processing the request + // and we should abort it by returning false here. + return true; + } // Unable to serve the request struct str response; @@ -819,11 +1345,14 @@ request_handler_json_rpc_try_handle const char *content_type = str_map_find (headers, "CONTENT_TYPE"); const char *method = str_map_find (headers, "REQUEST_METHOD"); - if (strcmp (method, "POST") - || !validate_json_rpc_content_type (content_type)) + if (!method || strcmp (method, "POST") + || !content_type || !validate_json_rpc_content_type (content_type)) return false; - // TODO: install the handler, perhaps construct an object + struct str *buf = xcalloc (1, sizeof *buf); + str_init (buf); + + request->handler_data = buf; return true; } @@ -831,15 +1360,28 @@ static bool request_handler_json_rpc_push (struct request *request, const void *data, size_t len) { - // TODO: append to a buffer - // TODO: len == 0: process the request + struct str *buf = request->handler_data; + if (len) + str_append_data (buf, data, len); + else + { + struct str response; + str_init (&response); + process_json_rpc (request->ctx, buf->str, buf->len, &response); + request->write_cb (request->user_data, response.str, response.len); + str_free (&response); + } return true; } static void request_handler_json_rpc_destroy (struct request *request) { - // TODO + struct str *buf = request->handler_data; + str_free (buf); + free (buf); + + request->handler_data = NULL; } struct request_handler g_request_handler_json_rpc = @@ -849,6 +1391,8 @@ struct request_handler g_request_handler_json_rpc = .destroy_cb = request_handler_json_rpc_destroy, }; +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + // TODO: another request handler to respond to all GETs with a message // --- Client communication handlers ------------------------------------------- @@ -927,17 +1471,86 @@ client_remove (struct client *client) struct client_fcgi { - struct fcgi_parser parser; ///< FastCGI stream parser + struct fcgi_muxer muxer; ///< FastCGI de/multiplexer }; +struct client_fcgi_request +{ + struct fcgi_request *fcgi_request; ///< FastCGI request + struct request request; ///< Request +}; + +static void +client_fcgi_request_write (void *user_data, const void *data, size_t len) +{ + struct client_fcgi_request *request = user_data; + fcgi_request_write (request->fcgi_request, data, len); +} + +static void +client_fcgi_request_close (void *user_data) +{ + struct client_fcgi_request *request = user_data; + // TODO: tell the fcgi_request to what? +} + +static void * +client_fcgi_request_start (void *user_data, struct fcgi_request *fcgi_request) +{ + struct client *client = user_data; + + struct client_fcgi_request *request = xmalloc (sizeof *request); + request->fcgi_request = fcgi_request; + request_init (&request->request); + request->request.ctx = client->ctx; + request->request.write_cb = client_fcgi_request_write; + request->request.close_cb = client_fcgi_request_close; + request->request.user_data = request; + return request; +} + +static void +client_fcgi_request_push (void *handler_data, const void *data, size_t len) +{ + struct client_fcgi_request *request = handler_data; + request_push (&request->request, data, len); +} + +static void +client_fcgi_request_destroy (void *handler_data) +{ + struct client_fcgi_request *request = handler_data; + request_free (&request->request); + free (handler_data); +} + +static void +client_fcgi_write (void *user_data, const void *data, size_t len) +{ + struct client *client = user_data; + client_write (client, data, len); +} + +static void +client_fcgi_close (void *user_data) +{ + struct client *client = user_data; + client_remove (client); +} + static void client_fcgi_init (struct client *client) { struct client_fcgi *self = xcalloc (1, sizeof *self); client->impl_data = self; - fcgi_parser_init (&self->parser); - // TODO: configure the parser + fcgi_muxer_init (&self->muxer); + self->muxer.write_cb = client_fcgi_write; + self->muxer.close_cb = client_fcgi_close; + self->muxer.request_start_cb = client_fcgi_request_start; + self->muxer.request_push_cb = client_fcgi_request_push; + self->muxer.request_destroy_cb = client_fcgi_request_destroy; + self->muxer.user_data = client; } static void @@ -946,7 +1559,7 @@ client_fcgi_destroy (struct client *client) struct client_fcgi *self = client->impl_data; client->impl_data = NULL; - fcgi_parser_free (&self->parser); + fcgi_muxer_free (&self->muxer); free (self); } @@ -954,7 +1567,7 @@ static bool client_fcgi_push (struct client *client, const void *data, size_t len) { struct client_fcgi *self = client->impl_data; - fcgi_parser_push (&self->parser, data, len); + fcgi_muxer_push (&self->muxer, data, len); return true; } @@ -1015,6 +1628,7 @@ client_scgi_init (struct client *client) client->impl_data = self; request_init (&self->request); + self->request.ctx = client->ctx; self->request.write_cb = client_scgi_write; self->request.close_cb = client_scgi_close; self->request.user_data = client; @@ -1426,6 +2040,8 @@ main (int argc, char *argv[]) (void) signal (SIGPIPE, SIG_IGN); + LIST_PREPEND (ctx.handlers, &g_request_handler_json_rpc); + if (!parse_config (&ctx, &e) || !setup_listen_fds (&ctx, &e)) { From 931fc441f6b89e8fed9c912ddb99d193f99f8b56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Sun, 8 Mar 2015 09:41:10 +0100 Subject: [PATCH 06/79] Steady progress Added static content serving with sane content type detection. Started working on WebSockets (meanwhile neither SCGI or FastCGI is finished and almost nothing has been tested). --- .gitmodules | 3 + CMakeLists.txt | 7 +- README | 9 +- cmake/FindLibMagic.cmake | 6 ++ demo-json-rpc-server.c | 192 +++++++++++++++++++++++++++++++++++++-- http-parser | 1 + 6 files changed, 206 insertions(+), 12 deletions(-) create mode 100644 cmake/FindLibMagic.cmake create mode 160000 http-parser diff --git a/.gitmodules b/.gitmodules index 5abb60c..c8d5acf 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "liberty"] path = liberty url = git://github.com/pjanouch/liberty.git +[submodule "http-parser"] + path = http-parser + url = https://github.com/joyent/http-parser.git diff --git a/CMakeLists.txt b/CMakeLists.txt index ca9c0ea..6540894 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,9 +23,12 @@ set (CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake) find_package (PkgConfig REQUIRED) pkg_check_modules (dependencies REQUIRED jansson) find_package (LibEV REQUIRED) +find_package (LibMagic REQUIRED) -set (project_libraries ${dependencies_LIBRARIES} ${LIBEV_LIBRARIES}) -include_directories (${dependencies_INCLUDE_DIRS} ${LIBEV_INCLUDE_DIRS}) +set (project_libraries ${dependencies_LIBRARIES} + ${LIBEV_LIBRARIES} ${LIBMAGIC_LIBRARIES}) +include_directories (${dependencies_INCLUDE_DIRS} + ${LIBEV_INCLUDE_DIRS} ${LIBMAGIC_INCLUDE_DIRS}) # Generate a configuration file configure_file (${PROJECT_SOURCE_DIR}/config.h.in ${PROJECT_BINARY_DIR}/config.h) diff --git a/README b/README index 311e388..f845a5d 100644 --- a/README +++ b/README @@ -2,13 +2,15 @@ acid ==== `acid' is A Continuous Integration Daemon. Currently under heavy development. +Right now I'm working on a demo JSON-RPC server that will serve as the basis for +the final daemon. The aim of this project is to provide a dumbed-down alternative to Travis CI. I find it way too complex to set up and run in a local setting, while the basic gist of it is actually very simple -- run some stuff on new git commits. -`acid' will provide a JSON-RPC 2.0 service for frontends over FastCGI or SCGI, -as well as a webhook endpoint for notifications about new commits. +`acid' will provide a JSON-RPC 2.0 service for frontends over FastCGI, SCGI, or +WebSockets, as well as a webhook endpoint for notifications about new commits. `acid' will be able to tell you about build results via e-mail and/or IRC. @@ -21,7 +23,8 @@ in a somewhat clean manner. Feel free to contribute. Building and Installing ----------------------- -Build dependencies: CMake, pkg-config, help2man, liberty (included) +Build dependencies: CMake, pkg-config, help2man, libmagic, + liberty (included), http-parser (included) Runtime dependencies: libev, Jansson $ git clone https://github.com/pjanouch/acid.git diff --git a/cmake/FindLibMagic.cmake b/cmake/FindLibMagic.cmake new file mode 100644 index 0000000..edae3bf --- /dev/null +++ b/cmake/FindLibMagic.cmake @@ -0,0 +1,6 @@ +# Public Domain + +find_library (LIBMAGIC_LIBRARIES magic) +find_path (LIBMAGIC_INCLUDE_DIRS magic.h) +find_package_handle_standard_args (LibMagic DEFAULT_MSG + LIBMAGIC_LIBRARIES LIBMAGIC_INCLUDE_DIRS) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index 6be6b3e..25d6eaa 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -34,6 +34,7 @@ #include #include +#include // --- Extensions to liberty --------------------------------------------------- @@ -733,7 +734,7 @@ fcgi_muxer_on_abort_request return; } - // TODO: abort the request + // TODO: abort the request: let it somehow produce FCGI_END_REQUEST } static void @@ -995,6 +996,8 @@ static struct config_item g_config_table[] = { "bind_host", NULL, "Address of the server" }, { "port_fastcgi", "9000", "Port to bind for FastCGI" }, { "port_scgi", NULL, "Port to bind for SCGI" }, + { "port_ws", NULL, "Port to bind for WebSockets" }, + { "static_root", NULL, "The root for static content" }, { NULL, NULL, NULL } }; @@ -1146,9 +1149,6 @@ json_rpc_ping (struct server_context *ctx, json_t *params) static json_t * process_json_rpc_request (struct server_context *ctx, json_t *request) { - // TODO: takes a parsed JSON request and returns back a JSON reply. - // This function may get called multiple times for batch requests. - if (!json_is_object (request)) return json_rpc_response (NULL, NULL, json_rpc_error (JSON_RPC_ERROR_INVALID_REQUEST, NULL)); @@ -1183,7 +1183,6 @@ process_json_rpc_request (struct server_context *ctx, json_t *request) return response; // Notifications don't get responses - // TODO: separate notifications from non-notifications? json_decref (response); return NULL; } @@ -1320,7 +1319,8 @@ request_start (struct request *self, struct str_map *headers) // Unable to serve the request struct str response; str_init (&response); - str_append (&response, "404 Not Found\r\n\r\n"); + str_append (&response, "Status: 404 Not Found\n"); + str_append (&response, "Content-Type: text/plain\n\n"); self->write_cb (self->user_data, response.str, response.len); str_free (&response); return false; @@ -1393,7 +1393,147 @@ struct request_handler g_request_handler_json_rpc = // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -// TODO: another request handler to respond to all GETs with a message +// TODO: refactor this spaghetti-tier code +static bool +request_handler_static_try_handle + (struct request *request, struct str_map *headers) +{ + struct server_context *ctx = request->ctx; + const char *root = str_map_find (&ctx->config, "static_root"); + if (!root) + { + print_debug ("static document root not configured"); + return false; + } + + const char *method = str_map_find (headers, "REQUEST_METHOD"); + if (!method || strcmp (method, "GET")) + return false; + + // TODO: look at , REQUEST_URI in the headers + const char *path_info = str_map_find (headers, "PATH_INFO"); + if (!path_info) + { + print_debug ("PATH_INFO not defined"); + return false; + } + + struct str_vector v; + str_vector_init (&v); + split_str_ignore_empty (path_info, '/', &v); + + struct str_vector resolved; + str_vector_init (&resolved); + + // So that the joined path begins with a slash + str_vector_add (&resolved, ""); + + // We need to filter the path to stay in our root + // Being able to read /etc/passwd would be rather embarrasing + for (size_t i = 0; i < v.len; i++) + { + const char *dir = v.vector[i]; + if (!strcmp (dir, ".")) + continue; + + if (strcmp (dir, "..")) + str_vector_add (&resolved, dir); + else if (resolved.len) + str_vector_remove (&resolved, resolved.len - 1); + } + str_vector_free (&v); + + char *suffix = join_str_vector (&resolved, '/'); + str_vector_free (&resolved); + + char *path = xstrdup_printf ("%s%s", root, suffix); + + FILE *fp = fopen (path, "rb"); + if (!fp) + { + struct str response; + str_init (&response); + str_append (&response, "Status: 404 Not Found\n"); + str_append (&response, "Content-Type: text/plain\n\n"); + str_append_printf (&response, + "File %s was not found on this server\n", suffix); + request->write_cb (request->user_data, response.str, response.len); + str_free (&response); + + free (suffix); + free (path); + return false; + } + + free (suffix); + free (path); + + uint8_t buf[8192]; + size_t len; + + // Try to detect the Content-Type from the actual contents + char *mime_type = NULL; + if ((len = fread (buf, 1, sizeof buf, fp))) + { + magic_t cookie; + const char *magic = NULL; + if ((cookie = magic_open (MAGIC_MIME))) + { + if (!magic_load (cookie, NULL) + && (magic = magic_buffer (cookie, buf, len))) + mime_type = xstrdup (magic); + magic_close (cookie); + } + } + if (!mime_type) + { + print_debug ("MIME type detection failed"); + mime_type = xstrdup ("application/octet_stream"); + } + + struct str response; + str_init (&response); + str_append (&response, "Status: 200 OK\n"); + str_append_printf (&response, "Content-Type: %s\n\n", mime_type); + request->write_cb (request->user_data, response.str, response.len); + str_free (&response); + free (mime_type); + + // Write the chunk we've used to help us with magic detection; + // obviously we have to do it after we've written the headers + if (len) + request->write_cb (request->user_data, buf, len); + + while ((len = fread (buf, 1, sizeof buf, fp))) + request->write_cb (request->user_data, buf, len); + fclose (fp); + return true; +} + +static bool +request_handler_static_push + (struct request *request, const void *data, size_t len) +{ + (void) request; + (void) data; + (void) len; + + // Ignoring all content; we shouldn't receive any (GET) + return false; +} + +static void +request_handler_static_destroy (struct request *request) +{ + (void) request; +} + +struct request_handler g_request_handler_static = +{ + .try_handle = request_handler_static_try_handle, + .push_cb = request_handler_static_push, + .destroy_cb = request_handler_static_destroy, +}; // --- Client communication handlers ------------------------------------------- @@ -1673,6 +1813,35 @@ static struct client_impl g_client_scgi = .push = client_scgi_push, }; +// - - WebSockets - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +static void +client_ws_init (struct client *client) +{ + // TODO +} + +static void +client_ws_destroy (struct client *client) +{ + // TODO +} + +static bool +client_ws_push (struct client *client, const void *data, size_t len) +{ + // TODO: first push everything into a http_parser, then after a protocol + // upgrade start parsing the WebSocket frames themselves +} + +static struct client_impl g_client_ws = +{ + .init = client_ws_init, + .destroy = client_ws_destroy, + .push = client_ws_push, +}; + + // --- Basic server stuff ------------------------------------------------------ struct listener @@ -1874,6 +2043,7 @@ setup_listen_fds (struct server_context *ctx, struct error **e) const char *bind_host = str_map_find (&ctx->config, "bind_host"); const char *port_fcgi = str_map_find (&ctx->config, "port_fastcgi"); const char *port_scgi = str_map_find (&ctx->config, "port_scgi"); + const char *port_ws = str_map_find (&ctx->config, "port_ws"); struct addrinfo gai_hints; memset (&gai_hints, 0, sizeof gai_hints); @@ -1883,11 +2053,14 @@ setup_listen_fds (struct server_context *ctx, struct error **e) struct str_vector ports_fcgi; str_vector_init (&ports_fcgi); struct str_vector ports_scgi; str_vector_init (&ports_scgi); + struct str_vector ports_ws; str_vector_init (&ports_ws); if (port_fcgi) split_str_ignore_empty (port_fcgi, ',', &ports_fcgi); if (port_scgi) split_str_ignore_empty (port_scgi, ',', &ports_scgi); + if (port_ws) + split_str_ignore_empty (port_ws, ',', &ports_ws); size_t n_ports = ports_fcgi.len + ports_scgi.len; ctx->listeners = xcalloc (n_ports, sizeof *ctx->listeners); @@ -1898,9 +2071,13 @@ setup_listen_fds (struct server_context *ctx, struct error **e) for (size_t i = 0; i < ports_scgi.len; i++) listener_add (ctx, bind_host, ports_scgi.vector[i], &gai_hints, &g_client_scgi); + for (size_t i = 0; i < ports_ws.len; i++) + listener_add (ctx, bind_host, ports_ws.vector[i], + &gai_hints, &g_client_ws); str_vector_free (&ports_fcgi); str_vector_free (&ports_scgi); + str_vector_free (&ports_ws); if (!ctx->n_listeners) { @@ -2040,6 +2217,7 @@ main (int argc, char *argv[]) (void) signal (SIGPIPE, SIG_IGN); + LIST_PREPEND (ctx.handlers, &g_request_handler_static); LIST_PREPEND (ctx.handlers, &g_request_handler_json_rpc); if (!parse_config (&ctx, &e) diff --git a/http-parser b/http-parser new file mode 160000 index 0000000..d547f3b --- /dev/null +++ b/http-parser @@ -0,0 +1 @@ +Subproject commit d547f3b1a98ed07fdcdaf401a8cbc5fffe9bfa6c From 5885d1aa69ea10af445713edba0b632a7892f317 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Mon, 9 Mar 2015 23:32:01 +0100 Subject: [PATCH 07/79] Some intial WebSockets code --- CMakeLists.txt | 5 +- README | 2 + demo-json-rpc-server.c | 523 ++++++++++++++++++++++++++++++++++++++++- 3 files changed, 523 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6540894..c9c5762 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,12 +22,13 @@ set (CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake) # Dependencies find_package (PkgConfig REQUIRED) pkg_check_modules (dependencies REQUIRED jansson) +pkg_check_modules (libssl REQUIRED libssl libcrypto) find_package (LibEV REQUIRED) find_package (LibMagic REQUIRED) -set (project_libraries ${dependencies_LIBRARIES} +set (project_libraries ${dependencies_LIBRARIES} ${libssl_LIBRARIES} ${LIBEV_LIBRARIES} ${LIBMAGIC_LIBRARIES}) -include_directories (${dependencies_INCLUDE_DIRS} +include_directories (${dependencies_INCLUDE_DIRS} ${libssl_INCLUDE_DIRS} ${LIBEV_INCLUDE_DIRS} ${LIBMAGIC_INCLUDE_DIRS}) # Generate a configuration file diff --git a/README b/README index f845a5d..40594e4 100644 --- a/README +++ b/README @@ -11,6 +11,8 @@ gist of it is actually very simple -- run some stuff on new git commits. `acid' will provide a JSON-RPC 2.0 service for frontends over FastCGI, SCGI, or WebSockets, as well as a webhook endpoint for notifications about new commits. +The daemon is supposed to be "firewalled" by a normal HTTP server and it will +not provide TLS support to secure the communications. `acid' will be able to tell you about build results via e-mail and/or IRC. diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index 25d6eaa..19a12ca 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -24,6 +24,8 @@ #define print_status_data ((void *) LOG_INFO) #define print_debug_data ((void *) LOG_DEBUG) +#define LIBERTY_WANT_SSL + #include "config.h" #include "liberty/liberty.c" @@ -36,6 +38,10 @@ #include #include +// FIXME: don't include the implementation, include the header and compile +// the implementation separately +#include "http-parser/http_parser.c" + // --- Extensions to liberty --------------------------------------------------- // These should be incorporated into the library ASAP @@ -106,6 +112,48 @@ str_pack_u64 (struct str *self, uint64_t x) str_append_data (self, tmp, sizeof tmp); } +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +static void +base64_encode (const void *data, size_t len, struct str *output) +{ + const char *alphabet = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + + const uint8_t *p = data; + size_t n_groups = len / 3; + size_t tail = len - n_groups * 3; + uint32_t group; + + for (; n_groups--; p += 3) + { + group = p[0] << 16 | p[1] << 8 | p[2]; + str_append_c (output, alphabet[(group >> 18) & 63]); + str_append_c (output, alphabet[(group >> 12) & 63]); + str_append_c (output, alphabet[(group >> 6) & 63]); + str_append_c (output, alphabet[ group & 63]); + } + + switch (tail) + { + case 2: + group = p[0] << 16 | p[1] << 8; + str_append_c (output, alphabet[(group >> 18) & 63]); + str_append_c (output, alphabet[(group >> 12) & 63]); + str_append_c (output, alphabet[(group >> 6) & 63]); + str_append_c (output, '='); + break; + case 1: + group = p[0] << 16; + str_append_c (output, alphabet[(group >> 18) & 63]); + str_append_c (output, alphabet[(group >> 12) & 63]); + str_append_c (output, '='); + str_append_c (output, '='); + default: + break; + } +} + // --- libev helpers ----------------------------------------------------------- static bool @@ -989,6 +1037,438 @@ scgi_parser_push (struct scgi_parser *self, return false; } +// --- WebSockets -------------------------------------------------------------- + +#define WS_GUID "258EAFA5-E914-47DA-95CA-C5AB0DC85B11" + +#define SEC_WS_KEY "Sec-WebSocket-Key" +#define SEC_WS_ACCEPT "Sec-WebSocket-Accept" +#define SEC_WS_PROTOCOL "Sec-WebSocket-Protocol" +#define SEC_WS_VERSION "Sec-WebSocket-Version" + +static char * +ws_encode_response_key (const char *key) +{ + char *response_key = xstrdup_printf ("%s" WS_GUID, key); + unsigned char hash[SHA_DIGEST_LENGTH]; + SHA1 ((unsigned char *) response_key, strlen (response_key), hash); + free (response_key); + + struct str base64; + str_init (&base64); + base64_encode (hash, sizeof hash, &base64); + return str_steal (&base64); +} + +enum ws_status +{ + // These names aren't really standard, just somewhat descriptive. + // The RFC isn't really much cleaner about their meaning. + + WS_STATUS_NORMAL = 1000, + WS_STATUS_GOING_AWAY = 1001, + WS_STATUS_PROTOCOL = 1002, + WS_STATUS_UNACCEPTABLE = 1003, + WS_STATUS_INCONSISTENT = 1007, + WS_STATUS_POLICY = 1008, + WS_STATUS_TOO_BIG = 1009, + WS_STATUS_EXTENSION = 1010, + WS_STATUS_UNEXPECTED = 1011, + + // Reserved for internal usage + WS_STATUS_MISSING = 1005, + WS_STATUS_ABNORMAL = 1006, + WS_STATUS_TLS = 1015 +}; + +// - - Frame parser - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +enum ws_parser_state +{ + WS_PARSER_FIXED, ///< Parsing fixed length part + WS_PARSER_PAYLOAD_LEN_16, ///< Parsing extended payload length + WS_PARSER_PAYLOAD_LEN_64, ///< Parsing extended payload length + WS_PARSER_MASK, ///< Parsing masking-key + WS_PARSER_PAYLOAD ///< Parsing payload +}; + +enum ws_opcode +{ + // Non-control + WS_OPCODE_CONT = 0, + WS_OPCODE_TEXT = 1, + WS_OPCODE_BINARY = 2, + + // Control + WS_OPCODE_CLOSE = 8, + WS_OPCODE_PING = 9, + WS_OPCODE_PONG = 10 +}; + +struct ws_parser +{ + struct str input; ///< External input buffer + enum ws_parser_state state; ///< Parsing state + + unsigned is_fin : 1; ///< Final frame of a message? + unsigned is_masked : 1; ///< Is the frame masked? + unsigned reserved_1 : 1; ///< Reserved + unsigned reserved_2 : 1; ///< Reserved + unsigned reserved_3 : 1; ///< Reserved + enum ws_opcode opcode; ///< Opcode + uint32_t mask; ///< Frame mask + uint64_t payload_len; ///< Payload length + + // TODO: it wouldn't be half bad if there was a callback to just validate + // the frame header (such as the maximum payload length) + + /// Callback for when a message is successfully parsed. + /// The actual payload is stored in "input", of length "payload_len". + bool (*on_frame) (void *user_data, const struct ws_parser *self); + + void *user_data; ///< User data for callbacks +}; + +static void +ws_parser_init (struct ws_parser *self) +{ + memset (self, 0, sizeof *self); + str_init (&self->input); +} + +static void +ws_parser_free (struct ws_parser *self) +{ + str_free (&self->input); +} + +static void +ws_parser_demask (struct ws_parser *self) +{ + // Yes, this could be made faster. For example by reading the mask in + // native byte ordering and applying it directly here. + + uint64_t end = self->payload_len & ~(uint64_t) 3; + for (uint64_t i = 0; i < end; i += 4) + { + self->input.str[i + 3] ^= self->mask & 0xFF; + self->input.str[i + 2] ^= (self->mask >> 8) & 0xFF; + self->input.str[i + 1] ^= (self->mask >> 16) & 0xFF; + self->input.str[i ] ^= (self->mask >> 24) & 0xFF; + } + + switch (self->payload_len - end) + { + case 3: + self->input.str[end + 2] ^= (self->mask >> 8) & 0xFF; + case 2: + self->input.str[end + 1] ^= (self->mask >> 16) & 0xFF; + case 1: + self->input.str[end ] ^= (self->mask >> 24) & 0xFF; + break; + } +} + +static bool +ws_parser_push (struct ws_parser *self, const void *data, size_t len) +{ + str_append_data (&self->input, data, len); + + struct msg_unpacker unpacker; + msg_unpacker_init (&unpacker, self->input.str, self->input.len); + + while (true) + switch (self->state) + { + uint8_t u8; + uint16_t u16; + + case WS_PARSER_FIXED: + if (self->input.len < 2) + return true; + + (void) msg_unpacker_u8 (&unpacker, &u8); + self->is_fin = (u8 >> 7) & 1; + self->reserved_1 = (u8 >> 6) & 1; + self->reserved_2 = (u8 >> 5) & 1; + self->reserved_3 = (u8 >> 4) & 1; + self->opcode = u8 & 15; + + (void) msg_unpacker_u8 (&unpacker, &u8); + self->is_masked = (u8 >> 7) & 1; + self->payload_len = u8 & 127; + + if (self->payload_len == 127) + self->state = WS_PARSER_PAYLOAD_LEN_64; + else if (self->payload_len == 126) + self->state = WS_PARSER_PAYLOAD_LEN_16; + else + self->state = self->is_masked + ? WS_PARSER_MASK + : WS_PARSER_PAYLOAD; + + str_remove_slice (&self->input, 0, 2); + break; + + case WS_PARSER_PAYLOAD_LEN_16: + if (self->input.len < 2) + return true; + + (void) msg_unpacker_u16 (&unpacker, &u16); + self->payload_len = u16; + + self->state = self->is_masked + ? WS_PARSER_MASK + : WS_PARSER_PAYLOAD; + str_remove_slice (&self->input, 0, 2); + break; + + case WS_PARSER_PAYLOAD_LEN_64: + if (self->input.len < 8) + return true; + + (void) msg_unpacker_u64 (&unpacker, &self->payload_len); + + self->state = self->is_masked + ? WS_PARSER_MASK + : WS_PARSER_PAYLOAD; + str_remove_slice (&self->input, 0, 8); + break; + + case WS_PARSER_MASK: + if (self->input.len < 4) + return true; + + (void) msg_unpacker_u32 (&unpacker, &self->mask); + + self->state = WS_PARSER_PAYLOAD; + str_remove_slice (&self->input, 0, 4); + break; + + case WS_PARSER_PAYLOAD: + if (self->input.len < self->payload_len) + return true; + + if (self->is_masked) + ws_parser_demask (self); + if (!self->on_frame (self->user_data, self)) + return false; + + self->state = WS_PARSER_FIXED; + str_reset (&self->input); + break; + } +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +// TODO: something to build frames for data + +// - - Server handler - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +// WebSockets aren't CGI-compatible, therefore we must handle the initial HTTP +// handshake ourselves. Luckily it's not too much of a bother with http-parser. +// Typically there will be a normal HTTP server in front of us, proxying the +// requests based on the URI. + +enum ws_handler_state +{ + WS_HANDLER_HTTP, ///< Parsing HTTP + WS_HANDLER_WEBSOCKETS ///< Parsing WebSockets frames +}; + +struct ws_handler +{ + enum ws_handler_state state; ///< State + + http_parser hp; ///< HTTP parser + bool parsing_header_value; ///< Parsing header value or field? + struct str field; ///< Field part buffer + struct str value; ///< Value part buffer + struct str_map headers; ///< HTTP Headers + struct str url; ///< Request URL + + struct ws_parser parser; ///< Protocol frame parser + + /// Called upon reception of a single full message + bool (*on_message) (void *user_data, const void *data, size_t len); + + /// Write a chunk of data to the stream + void (*write_cb) (void *user_data, const void *data, size_t len); + + // TODO: close_cb + + void *user_data; ///< User data for callbacks +}; + +static bool +ws_handler_on_frame (void *user_data, const struct ws_parser *parser) +{ + struct ws_handler *self = user_data; + // TODO: handle pings and what not + // TODO: validate the message + + return self->on_message (self->user_data, + self->parser.input.str, self->parser.payload_len); +} + +static void +ws_handler_init (struct ws_handler *self) +{ + memset (self, 0, sizeof *self); + + http_parser_init (&self->hp, HTTP_REQUEST); + self->hp.data = self; + + str_init (&self->field); + str_init (&self->value); + str_map_init (&self->headers); + self->headers.free = free; + // TODO: set headers.key_strxfrm? + str_init (&self->url); + + ws_parser_init (&self->parser); + self->parser.on_frame = ws_handler_on_frame; +} + +static void +ws_handler_free (struct ws_handler *self) +{ + str_free (&self->field); + str_free (&self->value); + str_map_free (&self->headers); + str_free (&self->url); + ws_parser_free (&self->parser); +} + +static void +ws_handler_on_header_read (struct ws_handler *self) +{ + // TODO: some headers can appear more than once, concatenate their values; + // for example "Sec-WebSocket-Version" + str_map_set (&self->headers, self->field.str, self->value.str); +} + +static int +ws_handler_on_header_field (http_parser *parser, const char *at, size_t len) +{ + struct ws_handler *self = parser->data; + if (self->parsing_header_value) + { + ws_handler_on_header_read (self); + str_reset (&self->field); + str_reset (&self->value); + } + str_append_data (&self->field, at, len); + self->parsing_header_value = false; + return 0; +} + +static int +ws_handler_on_header_value (http_parser *parser, const char *at, size_t len) +{ + struct ws_handler *self = parser->data; + str_append_data (&self->value, at, len); + self->parsing_header_value = true; + return 0; +} + +static int +ws_handler_on_headers_complete (http_parser *parser) +{ + // Just return 1 to tell the parser we don't want to parse any body; + // the parser should have found an upgrade request for WebSockets + (void) parser; + return 1; +} + +static int +ws_handler_on_url (http_parser *parser, const char *at, size_t len) +{ + struct ws_handler *self = parser->data; + str_append_data (&self->value, at, len); + return 0; +} + +static bool +ws_handler_finish_handshake (struct ws_handler *self) +{ + // TODO: probably factor this block out into its own function + // TODO: check if everything seems to be right + if (self->hp.method != HTTP_GET + || self->hp.http_major != 1 + || self->hp.http_minor != 1) + ; // TODO: error (maybe send a frame depending on conditions) + + const char *upgrade = str_map_find (&self->headers, "Upgrade"); + + const char *key = str_map_find (&self->headers, SEC_WS_KEY); + const char *version = str_map_find (&self->headers, SEC_WS_VERSION); + const char *protocol = str_map_find (&self->headers, SEC_WS_PROTOCOL); + + struct str response; + str_init (&response); + str_append (&response, "HTTP/1.1 101 Switching Protocols\r\n"); + str_append (&response, "Upgrade: websocket\r\n"); + str_append (&response, "Connection: Upgrade\r\n"); + + // TODO: prepare the rest of the headers + + // TODO: we should ideally check that this is a 16-byte base64-encoded + // value; do we also have to strip surrounding whitespace? + char *response_key = ws_encode_response_key (key); + str_append_printf (&response, SEC_WS_ACCEPT ": %s\r\n", response_key); + free (response_key); + + str_append (&response, "\r\n"); + self->write_cb (self->user_data, response.str, response.len); + str_free (&response); + return true; +} + +static bool +ws_handler_push (struct ws_handler *self, const void *data, size_t len) +{ + if (self->state == WS_HANDLER_WEBSOCKETS) + return ws_parser_push (&self->parser, data, len); + + // The handshake hasn't been done yet, process HTTP headers + static const http_parser_settings http_settings = + { + .on_header_field = ws_handler_on_header_field, + .on_header_value = ws_handler_on_header_value, + .on_headers_complete = ws_handler_on_headers_complete, + .on_url = ws_handler_on_url, + }; + + size_t n_parsed = http_parser_execute (&self->hp, + &http_settings, data, len); + + if (self->hp.upgrade) + { + if (len - n_parsed) + { + // TODO: error: the handshake hasn't been finished, yet there + // is more data to process after the headers + } + + if (!ws_handler_finish_handshake (self)) + return false; + + self->state = WS_HANDLER_WEBSOCKETS; + return true; + } + else if (n_parsed != len || HTTP_PARSER_ERRNO (&self->hp) != HPE_OK) + { + // TODO: error + // print_debug (..., http_errno_description + // (HTTP_PARSER_ERRNO (&self->hp)); + } + + // TODO: make double sure to handle the case of !upgrade + return true; +} + // --- Server ------------------------------------------------------------------ static struct config_item g_config_table[] = @@ -1736,7 +2216,7 @@ client_scgi_write (void *user_data, const void *data, size_t len) static void client_scgi_close (void *user_data) { - // XXX: this rather really means "close me [the request]" + // NOTE: this rather really means "close me [the request]" struct client *client = user_data; client_remove (client); } @@ -1815,23 +2295,56 @@ static struct client_impl g_client_scgi = // - - WebSockets - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +struct client_ws +{ + struct ws_handler handler; ///< WebSockets connection handler +}; + +static void +client_ws_write (void *user_data, const void *data, size_t len) +{ + struct client *client = user_data; + client_write (client, data, len); +} + +static bool +client_ws_on_message (void *user_data, const void *data, size_t len) +{ + struct client *client = user_data; + struct client_ws *self = client->impl_data; + + // TODO: do something about the message + return true; +} + static void client_ws_init (struct client *client) { - // TODO + struct client_ws *self = xmalloc (sizeof *self); + client->impl_data = self; + + ws_handler_init (&self->handler); + self->handler.write_cb = client_ws_write; + self->handler.on_message = client_ws_on_message; + self->handler.user_data = client; + // TODO: configure the handler some more, e.g. regarding the protocol } static void client_ws_destroy (struct client *client) { - // TODO + struct client_ws *self = client->impl_data; + client->impl_data = NULL; + + ws_handler_free (&self->handler); + free (self); } static bool client_ws_push (struct client *client, const void *data, size_t len) { - // TODO: first push everything into a http_parser, then after a protocol - // upgrade start parsing the WebSocket frames themselves + struct client_ws *self = client->impl_data; + return ws_handler_push (&self->handler, data, len); } static struct client_impl g_client_ws = From 3c0e48a429bf810b18de1cd2b9e3af1455e6dc7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Tue, 10 Mar 2015 20:48:25 +0100 Subject: [PATCH 08/79] Refactoring --- demo-json-rpc-server.c | 116 ++++++++++++++++++++--------------------- 1 file changed, 57 insertions(+), 59 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index 19a12ca..cdad2be 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -1203,9 +1203,7 @@ ws_parser_push (struct ws_parser *self, const void *data, size_t len) else if (self->payload_len == 126) self->state = WS_PARSER_PAYLOAD_LEN_16; else - self->state = self->is_masked - ? WS_PARSER_MASK - : WS_PARSER_PAYLOAD; + self->state = WS_PARSER_MASK; str_remove_slice (&self->input, 0, 2); break; @@ -1217,9 +1215,7 @@ ws_parser_push (struct ws_parser *self, const void *data, size_t len) (void) msg_unpacker_u16 (&unpacker, &u16); self->payload_len = u16; - self->state = self->is_masked - ? WS_PARSER_MASK - : WS_PARSER_PAYLOAD; + self->state = WS_PARSER_MASK; str_remove_slice (&self->input, 0, 2); break; @@ -1229,13 +1225,16 @@ ws_parser_push (struct ws_parser *self, const void *data, size_t len) (void) msg_unpacker_u64 (&unpacker, &self->payload_len); - self->state = self->is_masked - ? WS_PARSER_MASK - : WS_PARSER_PAYLOAD; + self->state = WS_PARSER_MASK; str_remove_slice (&self->input, 0, 8); break; case WS_PARSER_MASK: + if (!self->is_masked) + { + self->state = WS_PARSER_PAYLOAD; + break; + } if (self->input.len < 4) return true; @@ -2416,52 +2415,53 @@ error: } static void -on_client_available (EV_P_ ev_io *watcher, int revents) +make_client (EV_P_ struct client_impl *impl, int sock_fd) { struct server_context *ctx = ev_userdata (loop); + set_blocking (sock_fd, false); + + struct client *client = xmalloc (sizeof *client); + client_init (client); + client->socket_fd = sock_fd; + client->impl = impl; + + ev_io_init (&client->read_watcher, on_client_ready, sock_fd, EV_READ); + ev_io_init (&client->write_watcher, on_client_ready, sock_fd, EV_WRITE); + client->read_watcher.data = client; + client->write_watcher.data = client; + + // We're only interested in reading as the write queue is empty now + ev_io_start (EV_A_ &client->read_watcher); + + // Initialize the higher-level implementation + client->impl->init (client); + + LIST_PREPEND (ctx->clients, client); + ctx->n_clients++; +} + +static void +on_client_available (EV_P_ ev_io *watcher, int revents) +{ struct listener *listener = watcher->data; (void) revents; while (true) { int sock_fd = accept (watcher->fd, NULL, NULL); - if (sock_fd == -1) - { - if (errno == EAGAIN) - break; - if (errno == EINTR - || errno == ECONNABORTED) - continue; - - // Stop accepting connections to prevent busy looping - ev_io_stop (EV_A_ watcher); - - print_fatal ("%s: %s", "accept", strerror (errno)); - // TODO: initiate_quit (ctx); + if (sock_fd != -1) + make_client (EV_A_ listener->impl, sock_fd); + else if (errno == EAGAIN) + return; + else if (errno != EINTR && errno != ECONNABORTED) break; - } - - set_blocking (sock_fd, false); - - struct client *client = xmalloc (sizeof *client); - client_init (client); - client->socket_fd = sock_fd; - client->impl = listener->impl; - - ev_io_init (&client->read_watcher, on_client_ready, sock_fd, EV_READ); - ev_io_init (&client->write_watcher, on_client_ready, sock_fd, EV_WRITE); - client->read_watcher.data = client; - client->write_watcher.data = client; - - // We're only interested in reading as the write queue is empty now - ev_io_start (EV_A_ &client->read_watcher); - - // Initialize the higher-level implementation - client->impl->init (client); - - LIST_PREPEND (ctx->clients, client); - ctx->n_clients++; } + + // Stop accepting connections to prevent busy looping + ev_io_stop (EV_A_ watcher); + + print_fatal ("%s: %s", "accept", strerror (errno)); + // TODO: initiate_quit (ctx); } // --- Application setup ------------------------------------------------------- @@ -2520,7 +2520,7 @@ listener_finish (struct addrinfo *gai_iter) static void listener_add (struct server_context *ctx, const char *host, const char *port, - struct addrinfo *gai_hints, struct client_impl *impl) + const struct addrinfo *gai_hints, struct client_impl *impl) { struct addrinfo *gai_result, *gai_iter; int err = getaddrinfo (host, port, gai_hints, &gai_result); @@ -2553,29 +2553,27 @@ listener_add (struct server_context *ctx, const char *host, const char *port, static bool setup_listen_fds (struct server_context *ctx, struct error **e) { + static const struct addrinfo gai_hints = + { + .ai_socktype = SOCK_STREAM, + .ai_flags = AI_PASSIVE, + }; + const char *bind_host = str_map_find (&ctx->config, "bind_host"); + const char *port_fcgi = str_map_find (&ctx->config, "port_fastcgi"); const char *port_scgi = str_map_find (&ctx->config, "port_scgi"); const char *port_ws = str_map_find (&ctx->config, "port_ws"); - struct addrinfo gai_hints; - memset (&gai_hints, 0, sizeof gai_hints); - - gai_hints.ai_socktype = SOCK_STREAM; - gai_hints.ai_flags = AI_PASSIVE; - struct str_vector ports_fcgi; str_vector_init (&ports_fcgi); struct str_vector ports_scgi; str_vector_init (&ports_scgi); struct str_vector ports_ws; str_vector_init (&ports_ws); - if (port_fcgi) - split_str_ignore_empty (port_fcgi, ',', &ports_fcgi); - if (port_scgi) - split_str_ignore_empty (port_scgi, ',', &ports_scgi); - if (port_ws) - split_str_ignore_empty (port_ws, ',', &ports_ws); + if (port_fcgi) split_str_ignore_empty (port_fcgi, ',', &ports_fcgi); + if (port_scgi) split_str_ignore_empty (port_scgi, ',', &ports_scgi); + if (port_ws) split_str_ignore_empty (port_ws, ',', &ports_ws); - size_t n_ports = ports_fcgi.len + ports_scgi.len; + size_t n_ports = ports_fcgi.len + ports_scgi.len + ports_ws.len; ctx->listeners = xcalloc (n_ports, sizeof *ctx->listeners); for (size_t i = 0; i < ports_fcgi.len; i++) From 012a57b357f7cc5a3ecd725d2cd0096aa8762b55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Wed, 11 Mar 2015 00:24:20 +0100 Subject: [PATCH 09/79] Steady progress Some further refactoring, added a few comments, etc. It's not about adding huge chunks of code anymore, and I'm slowly moving towards getting the details right. There's still a ton of TODO items, though. --- demo-json-rpc-server.c | 261 ++++++++++++++++++++++++++--------------- 1 file changed, 166 insertions(+), 95 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index cdad2be..1eebe4d 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -114,6 +114,21 @@ str_pack_u64 (struct str *self, uint64_t x) // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +static int +tolower_ascii (int c) +{ + return c >= 'A' && c <= 'Z' ? c + ('a' - 'A') : c; +} + +static size_t +tolower_ascii_strxfrm (char *dest, const char *src, size_t n) +{ + size_t len = strlen (src); + while (n-- && (*dest++ = tolower_ascii (*src++))) + ; + return len; +} + static void base64_encode (const void *data, size_t len, struct str *output) { @@ -548,6 +563,8 @@ struct fcgi_muxer { struct fcgi_parser parser; ///< FastCGI message parser + // TODO: bool quitting; that causes us to reject all requests? + /// Requests assigned to request IDs // TODO: allocate this dynamically struct fcgi_request *requests[1 << 16]; @@ -1143,7 +1160,7 @@ ws_parser_free (struct ws_parser *self) } static void -ws_parser_demask (struct ws_parser *self) +ws_parser_unmask (struct ws_parser *self) { // Yes, this could be made faster. For example by reading the mask in // native byte ordering and applying it directly here. @@ -1249,7 +1266,7 @@ ws_parser_push (struct ws_parser *self, const void *data, size_t len) return true; if (self->is_masked) - ws_parser_demask (self); + ws_parser_unmask (self); if (!self->on_frame (self->user_data, self)) return false; @@ -1289,6 +1306,9 @@ struct ws_handler struct ws_parser parser; ///< Protocol frame parser + // TODO: bool closing; + // TODO: a configurable max_payload_len initialized by _init() + /// Called upon reception of a single full message bool (*on_message) (void *user_data, const void *data, size_t len); @@ -1306,7 +1326,7 @@ ws_handler_on_frame (void *user_data, const struct ws_parser *parser) struct ws_handler *self = user_data; // TODO: handle pings and what not // TODO: validate the message - + // TODO: first concatenate all parts of the message return self->on_message (self->user_data, self->parser.input.str, self->parser.payload_len); } @@ -1323,7 +1343,7 @@ ws_handler_init (struct ws_handler *self) str_init (&self->value); str_map_init (&self->headers); self->headers.free = free; - // TODO: set headers.key_strxfrm? + self->headers.key_xfrm = tolower_ascii_strxfrm; str_init (&self->url); ws_parser_init (&self->parser); @@ -1613,6 +1633,21 @@ validate_json_rpc_content_type (const char *type) // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +typedef json_t *(*json_rpc_handler_fn) (struct server_context *, json_t *); + +struct json_rpc_handler_info +{ + const char *method_name; ///< JSON-RPC method name + json_rpc_handler_fn handler; ///< Method handler +}; + +static int +json_rpc_handler_info_cmp (const void *first, const void *second) +{ + return strcmp (((struct json_rpc_handler_info *) first)->method_name, + ((struct json_rpc_handler_info *) second)->method_name); +} + // TODO: a method that queues up a ping over IRC: this has to be owned by the // server context as a background job that removes itself upon completion. @@ -1628,6 +1663,13 @@ json_rpc_ping (struct server_context *ctx, json_t *params) static json_t * process_json_rpc_request (struct server_context *ctx, json_t *request) { + // A list of all available methods; this list has to be ordered. + // Eventually it might be better to move this into a map in the context. + static struct json_rpc_handler_info handlers[] = + { + { "ping", json_rpc_ping }, + }; + if (!json_is_object (request)) return json_rpc_response (NULL, NULL, json_rpc_error (JSON_RPC_ERROR_INVALID_REQUEST, NULL)); @@ -1650,14 +1692,14 @@ process_json_rpc_request (struct server_context *ctx, json_t *request) return json_rpc_response (id, NULL, json_rpc_error (JSON_RPC_ERROR_INVALID_REQUEST, NULL)); - // TODO: add a more extensible mechanism - json_t *response = NULL; - if (!strcmp (method, "ping")) - response = json_rpc_ping (ctx, params); - else + struct json_rpc_handler_info key = { .method_name = method }; + struct json_rpc_handler_info *handler = bsearch (&key, handlers, + N_ELEMENTS (handlers), sizeof key, json_rpc_handler_info_cmp); + if (!handler) return json_rpc_response (id, NULL, json_rpc_error (JSON_RPC_ERROR_METHOD_NOT_FOUND, NULL)); + json_t *response = handler->handler (ctx, params); if (id) return response; @@ -1808,11 +1850,11 @@ request_start (struct request *self, struct str_map *headers) static bool request_push (struct request *self, const void *data, size_t len) { - if (soft_assert (self->handler)) - return self->handler->push_cb (self, data, len); + if (!soft_assert (self->handler)) + // No handler, nothing to do with any data + return false; - // No handler, nothing to do with any data - return false; + return self->handler->push_cb (self, data, len); } // --- Requests handlers ------------------------------------------------------- @@ -1841,16 +1883,19 @@ request_handler_json_rpc_push { struct str *buf = request->handler_data; if (len) - str_append_data (buf, data, len); - else { - struct str response; - str_init (&response); - process_json_rpc (request->ctx, buf->str, buf->len, &response); - request->write_cb (request->user_data, response.str, response.len); - str_free (&response); + str_append_data (buf, data, len); + return true; } - return true; + + struct str response; + str_init (&response); + str_append (&response, "Status: 200 OK\n"); + str_append_printf (&response, "Content-Type: %s\n\n", "application/json"); + process_json_rpc (request->ctx, buf->str, buf->len, &response); + request->write_cb (request->user_data, response.str, response.len); + str_free (&response); + return false; } static void @@ -1872,7 +1917,58 @@ struct request_handler g_request_handler_json_rpc = // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -// TODO: refactor this spaghetti-tier code +static char * +canonicalize_url_path (const char *path) +{ + struct str_vector v; + str_vector_init (&v); + split_str_ignore_empty (path, '/', &v); + + struct str_vector canonical; + str_vector_init (&canonical); + + // So that the joined path always begins with a slash + str_vector_add (&canonical, ""); + + for (size_t i = 0; i < v.len; i++) + { + const char *dir = v.vector[i]; + if (!strcmp (dir, ".")) + continue; + + if (strcmp (dir, "..")) + str_vector_add (&canonical, dir); + else if (canonical.len) + // ".." never goes above the root + str_vector_remove (&canonical, canonical.len - 1); + } + str_vector_free (&v); + + char *joined = join_str_vector (&canonical, '/'); + str_vector_free (&canonical); + return joined; +} + +static char * +detect_magic (const void *data, size_t len) +{ + magic_t cookie; + char *mime_type = NULL; + + if (!(cookie = magic_open (MAGIC_MIME))) + return NULL; + + const char *magic = NULL; + if (!magic_load (cookie, NULL) + && (magic = magic_buffer (cookie, data, len))) + mime_type = xstrdup (magic); + else + print_debug ("MIME type detection failed: %s", magic_error (cookie)); + + magic_close (cookie); + return mime_type; +} + static bool request_handler_static_try_handle (struct request *request, struct str_map *headers) @@ -1897,34 +1993,9 @@ request_handler_static_try_handle return false; } - struct str_vector v; - str_vector_init (&v); - split_str_ignore_empty (path_info, '/', &v); - - struct str_vector resolved; - str_vector_init (&resolved); - - // So that the joined path begins with a slash - str_vector_add (&resolved, ""); - // We need to filter the path to stay in our root // Being able to read /etc/passwd would be rather embarrasing - for (size_t i = 0; i < v.len; i++) - { - const char *dir = v.vector[i]; - if (!strcmp (dir, ".")) - continue; - - if (strcmp (dir, "..")) - str_vector_add (&resolved, dir); - else if (resolved.len) - str_vector_remove (&resolved, resolved.len - 1); - } - str_vector_free (&v); - - char *suffix = join_str_vector (&resolved, '/'); - str_vector_free (&resolved); - + char *suffix = canonicalize_url_path (path_info); char *path = xstrdup_printf ("%s%s", root, suffix); FILE *fp = fopen (path, "rb"); @@ -1953,22 +2024,9 @@ request_handler_static_try_handle // Try to detect the Content-Type from the actual contents char *mime_type = NULL; if ((len = fread (buf, 1, sizeof buf, fp))) - { - magic_t cookie; - const char *magic = NULL; - if ((cookie = magic_open (MAGIC_MIME))) - { - if (!magic_load (cookie, NULL) - && (magic = magic_buffer (cookie, buf, len))) - mime_type = xstrdup (magic); - magic_close (cookie); - } - } + mime_type = detect_magic (buf, len); if (!mime_type) - { - print_debug ("MIME type detection failed"); mime_type = xstrdup ("application/octet_stream"); - } struct str response; str_init (&response); @@ -2037,6 +2095,15 @@ struct client_impl /// Initialize the client as needed void (*init) (struct client *client); + // TODO: a method for graceful shutdown which will, in the case of + // WebSockets, actually send a "shutdown" close packet, and in the case + // of FastCGI will FCGI_END_REQUEST everything with FCGI_REQUEST_COMPLETE + // and FCGI_OVERLOADED all incoming requests in the meantime (the FastCGI + // specification isn't very clear about how we should respond to this). + // + // We then should set up a timer for about a second until we kill all + // clients for good. + /// Do any additional cleanup void (*destroy) (struct client *client); @@ -2370,28 +2437,23 @@ client_read_loop (EV_P_ struct client *client, ev_io *watcher) while (true) { ssize_t n_read = recv (watcher->fd, buf, sizeof buf, 0); - if (n_read < 0) + if (n_read >= 0) { - if (errno == EAGAIN) + if (!client->impl->push (client, buf, n_read)) + return false; + if (!n_read) break; - if (errno == EINTR) - continue; - - return false; } - - if (!client->impl->push (client, buf, n_read)) - return false; - - if (!n_read) - { - // Don't receive the EOF condition repeatedly - ev_io_stop (EV_A_ watcher); - - // We can probably still write, so let's just return + else if (errno == EAGAIN) return true; - } + else if (errno != EINTR) + return false; } + + // Don't receive the EOF condition repeatedly + ev_io_stop (EV_A_ watcher); + + // We can probably still write, so let's just return return true; } @@ -2402,15 +2464,18 @@ on_client_ready (EV_P_ ev_io *watcher, int revents) if (revents & EV_READ) if (!client_read_loop (EV_A_ client, watcher)) - goto error; + goto close; if (revents & EV_WRITE) + // TODO: shouldn't we at least provide an option (to be used by a client + // implementation if it so desires) to close the connection once we've + // finished flushing the write queue? This should probably even be + // the default behaviour, as it's fairly uncommon for clients to + // shutdown the socket for writes while leaving it open for reading. if (!flush_queue (&client->write_queue, watcher)) - goto error; + goto close; return; -error: - // The callback also could have just told us to stop reading, - // this is not necessarily an error condition +close: client_remove (client); } @@ -2479,7 +2544,7 @@ parse_config (struct server_context *ctx, struct error **e) } static int -listener_finish (struct addrinfo *gai_iter) +listener_bind (struct addrinfo *gai_iter) { int fd = socket (gai_iter->ai_family, gai_iter->ai_socktype, gai_iter->ai_protocol); @@ -2536,7 +2601,7 @@ listener_add (struct server_context *ctx, const char *host, const char *port, int fd; for (gai_iter = gai_result; gai_iter; gai_iter = gai_iter->ai_next) { - if ((fd = listener_finish (gai_iter)) == -1) + if ((fd = listener_bind (gai_iter)) == -1) continue; set_blocking (fd, false); @@ -2550,6 +2615,15 @@ listener_add (struct server_context *ctx, const char *host, const char *port, freeaddrinfo (gai_result); } +static void +get_ports_from_config (struct server_context *ctx, + const char *key, struct str_vector *out) +{ + const char *ports; + if ((ports = str_map_find (&ctx->config, key))) + split_str_ignore_empty (ports, ',', out); +} + static bool setup_listen_fds (struct server_context *ctx, struct error **e) { @@ -2559,20 +2633,15 @@ setup_listen_fds (struct server_context *ctx, struct error **e) .ai_flags = AI_PASSIVE, }; - const char *bind_host = str_map_find (&ctx->config, "bind_host"); - - const char *port_fcgi = str_map_find (&ctx->config, "port_fastcgi"); - const char *port_scgi = str_map_find (&ctx->config, "port_scgi"); - const char *port_ws = str_map_find (&ctx->config, "port_ws"); - struct str_vector ports_fcgi; str_vector_init (&ports_fcgi); struct str_vector ports_scgi; str_vector_init (&ports_scgi); struct str_vector ports_ws; str_vector_init (&ports_ws); - if (port_fcgi) split_str_ignore_empty (port_fcgi, ',', &ports_fcgi); - if (port_scgi) split_str_ignore_empty (port_scgi, ',', &ports_scgi); - if (port_ws) split_str_ignore_empty (port_ws, ',', &ports_ws); + get_ports_from_config (ctx, "port_fastcgi", &ports_fcgi); + get_ports_from_config (ctx, "port_scgi", &ports_scgi); + get_ports_from_config (ctx, "port_ws", &ports_ws); + const char *bind_host = str_map_find (&ctx->config, "bind_host"); size_t n_ports = ports_fcgi.len + ports_scgi.len + ports_ws.len; ctx->listeners = xcalloc (n_ports, sizeof *ctx->listeners); @@ -2615,6 +2684,8 @@ static void daemonize (void) { // TODO: create and lock a PID file? + // TODO: add the path for the PID file into "struct server_context", + // see the UNIX bible for more details on how to proceed. print_status ("daemonizing..."); if (chdir ("/")) From 43370388199efab39c98e110008e909e7e131250 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Wed, 11 Mar 2015 23:56:25 +0100 Subject: [PATCH 10/79] Try to lock a PID file --- demo-json-rpc-server.c | 55 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 4 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index 1eebe4d..1b10e9d 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -1496,6 +1496,7 @@ static struct config_item g_config_table[] = { "port_fastcgi", "9000", "Port to bind for FastCGI" }, { "port_scgi", NULL, "Port to bind for SCGI" }, { "port_ws", NULL, "Port to bind for WebSockets" }, + { "pid_file", NULL, "Full path for the PID file" }, { "static_root", NULL, "The root for static content" }, { NULL, NULL, NULL } }; @@ -2668,6 +2669,51 @@ setup_listen_fds (struct server_context *ctx, struct error **e) return true; } +static bool +lock_pid_file (struct server_context *ctx, struct error **e) +{ + const char *path = str_map_find (&ctx->config, "pid_file"); + if (!path) + return true; + + int fd = open (path, O_RDWR | O_CREAT, + S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH /* 644 */); + if (fd < 0) + { + error_set (e, "can't open `%s': %s", path, strerror (errno)); + return false; + } + + struct flock lock = + { + .l_type = F_WRLCK, + .l_start = 0, + .l_whence = SEEK_SET, + .l_len = 0, + }; + if (fcntl (fd, F_SETLK, &lock)) + { + error_set (e, "can't lock `%s': %s", path, strerror (errno)); + return false; + } + + struct str pid; + str_init (&pid); + str_append_printf (&pid, "%ld", (long) getpid ()); + + if (ftruncate (fd, 0) + || write (fd, pid.str, pid.len) != (ssize_t) pid.len) + { + error_set (e, "can't write to `%s': %s", path, strerror (errno)); + return false; + } + str_free (&pid); + + // Intentionally not closing the file descriptor; it must stay alive + // for the entire life of the application + return true; +} + // --- Main program ------------------------------------------------------------ static void @@ -2683,9 +2729,6 @@ on_termination_signal (EV_P_ ev_signal *handle, int revents) static void daemonize (void) { - // TODO: create and lock a PID file? - // TODO: add the path for the PID file into "struct server_context", - // see the UNIX bible for more details on how to proceed. print_status ("daemonizing..."); if (chdir ("/")) @@ -2708,7 +2751,10 @@ daemonize (void) openlog (PROGRAM_NAME, LOG_NDELAY | LOG_NOWAIT | LOG_PID, 0); g_log_message_real = log_message_syslog; - // XXX: we may close our own descriptors this way, crippling ourselves + // XXX: we may close our own descriptors this way, crippling ourselves; + // there is no real guarantee that we will start with all three + // descriptors open. In theory we could try to enumerate the descriptors + // at the start of main(). for (int i = 0; i < 3; i++) xclose (i); @@ -2803,6 +2849,7 @@ main (int argc, char *argv[]) LIST_PREPEND (ctx.handlers, &g_request_handler_json_rpc); if (!parse_config (&ctx, &e) + || !lock_pid_file (&ctx, &e) || !setup_listen_fds (&ctx, &e)) { print_error ("%s", e->message); From 23eb4cca38949152fb3f61dd9edd88e60cccf5a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Sat, 14 Mar 2015 19:36:37 +0100 Subject: [PATCH 11/79] Steady progress Still in a state of total chaos, it appears. --- demo-json-rpc-server.c | 320 +++++++++++++++++++++++++++++++++++------ 1 file changed, 275 insertions(+), 45 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index 1b10e9d..77c9ca5 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -701,6 +701,12 @@ fcgi_request_write (struct fcgi_request *self, const void *data, size_t len) } } +static void +fcgi_request_finish (struct fcgi_request *self) +{ + // TODO: flush(), end_request(), delete self, muxer->request_destroy_cb()? +} + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - typedef void (*fcgi_muxer_handler_fn) @@ -764,6 +770,7 @@ fcgi_muxer_on_begin_request return; } + // We can only act as a responder, reject everything else up front if (role != FCGI_RESPONDER) { fcgi_muxer_send_end_request (self, @@ -1063,6 +1070,8 @@ scgi_parser_push (struct scgi_parser *self, #define SEC_WS_PROTOCOL "Sec-WebSocket-Protocol" #define SEC_WS_VERSION "Sec-WebSocket-Version" +#define WS_MAX_CONTROL_PAYLOAD_LEN 125 + static char * ws_encode_response_key (const char *key) { @@ -1122,6 +1131,12 @@ enum ws_opcode WS_OPCODE_PONG = 10 }; +static bool +ws_is_control_frame (int opcode) +{ + return opcode >= WS_OPCODE_CLOSE; +} + struct ws_parser { struct str input; ///< External input buffer @@ -1136,8 +1151,7 @@ struct ws_parser uint32_t mask; ///< Frame mask uint64_t payload_len; ///< Payload length - // TODO: it wouldn't be half bad if there was a callback to just validate - // the frame header (such as the maximum payload length) + bool (*on_frame_header) (void *user_data, const struct ws_parser *self); /// Callback for when a message is successfully parsed. /// The actual payload is stored in "input", of length "payload_len". @@ -1248,17 +1262,17 @@ ws_parser_push (struct ws_parser *self, const void *data, size_t len) case WS_PARSER_MASK: if (!self->is_masked) - { - self->state = WS_PARSER_PAYLOAD; - break; - } + goto end_of_header; if (self->input.len < 4) return true; (void) msg_unpacker_u32 (&unpacker, &self->mask); - - self->state = WS_PARSER_PAYLOAD; str_remove_slice (&self->input, 0, 4); + + end_of_header: + self->state = WS_PARSER_PAYLOAD; + if (!self->on_frame_header (self->user_data, self)) + return false; break; case WS_PARSER_PAYLOAD: @@ -1289,7 +1303,7 @@ ws_parser_push (struct ws_parser *self, const void *data, size_t len) enum ws_handler_state { - WS_HANDLER_HTTP, ///< Parsing HTTP + WS_HANDLER_HANDSHAKE, ///< Parsing HTTP WS_HANDLER_WEBSOCKETS ///< Parsing WebSockets frames }; @@ -1305,12 +1319,38 @@ struct ws_handler struct str url; ///< Request URL struct ws_parser parser; ///< Protocol frame parser + bool expecting_continuation; ///< For non-control traffic - // TODO: bool closing; - // TODO: a configurable max_payload_len initialized by _init() + enum ws_opcode message_opcode; ///< Opcode for the current message + struct str message_data; ///< Concatenated message data + + unsigned ping_interval; ///< Ping interval in seconds + uint64_t max_payload_len; ///< Maximum length of any message + + // TODO: bool closing; // XXX: rather a { OPEN, CLOSING } state? + // TODO: a close timer + + // TODO: a ping timer (when no pong is received by the second time the + // timer triggers, it is a ping timeout) + ev_timer ping_timer; ///< Ping timer + bool received_pong; ///< Received PONG since the last PING /// Called upon reception of a single full message - bool (*on_message) (void *user_data, const void *data, size_t len); + bool (*on_message) (void *user_data, + enum ws_opcode type, const void *data, size_t len); + + // TODO: void (*on_initialized) () that will allow the user to choose + // any sub-protocol, if the client has provided any. + + /// The connection has been closed. + /// @a close_code may, or may not, be one of enum ws_status. + // NOTE: the "close_code" is what we receive from the remote endpoint, + // or one of 1005/1006/1015 + // NOTE: the reason is an empty string if omitted + // TODO; also note that ideally, the handler should (be able to) first + // receive a notification about the connection being closed because of + // an error (recv()) returns -1, and call on_close() in reaction. + void (*on_close) (void *user_data, int close_code, const char *reason); /// Write a chunk of data to the stream void (*write_cb) (void *user_data, const void *data, size_t len); @@ -1320,15 +1360,126 @@ struct ws_handler void *user_data; ///< User data for callbacks }; +static void +ws_handler_send_control (struct ws_handler *self, enum ws_opcode opcode, + const void *data, size_t len) +{ + if (len > WS_MAX_CONTROL_PAYLOAD_LEN) + { + print_debug ("truncating output control frame payload" + " from %zu to %zu bytes", len, (size_t) WS_MAX_CONTROL_PAYLOAD_LEN); + len = WS_MAX_CONTROL_PAYLOAD_LEN; + } + + uint8_t header[2] = { 0x80 | (opcode & 0x0F), len }; + self->write_cb (self->user_data, header, sizeof header); + self->write_cb (self->user_data, data, len); +} + +static void +ws_handler_fail (struct ws_handler *self, enum ws_status reason) +{ + uint8_t payload[2] = { reason << 8, reason }; + ws_handler_send_control (self, WS_OPCODE_CLOSE, payload, sizeof payload); + + // TODO: set the close timer, ignore all further incoming input (either set + // some flag for the case that we're in the middle of ws_handler_push(), + // and/or add a mechanism to stop the caller from polling the socket for + // reads). +} + +// TODO: ws_handler_close() that behaves like ws_handler_fail() but doesn't +// ignore frames up to a corresponding close from the client. +// Read the RFC once again to see if we can really process the frames. + +static bool +ws_handler_on_frame_header (void *user_data, const struct ws_parser *parser) +{ + struct ws_handler *self = user_data; + + if (parser->reserved_1 || parser->reserved_2 || parser->reserved_3 + || !parser->is_masked // client -> server payload must be masked + || (ws_is_control_frame (parser->opcode) && + (!parser->is_fin || parser->payload_len > WS_MAX_CONTROL_PAYLOAD_LEN)) + || (!ws_is_control_frame (parser->opcode) && + (self->expecting_continuation && parser->opcode != WS_OPCODE_CONT))) + ws_handler_fail (self, WS_STATUS_PROTOCOL); + else if (parser->payload_len > self->max_payload_len) + ws_handler_fail (self, WS_STATUS_TOO_BIG); + else + return true; + return false; +} + +static bool +ws_handler_on_control_frame + (struct ws_handler *self, const struct ws_parser *parser) +{ + switch (parser->opcode) + { + case WS_OPCODE_CLOSE: + // TODO: confirm the close + break; + case WS_OPCODE_PING: + ws_handler_send_control (self, WS_OPCODE_PONG, + parser->input.str, parser->payload_len); + break; + case WS_OPCODE_PONG: + // XXX: maybe we should check the payload + self->received_pong = true; + break; + default: + // TODO: shouldn't we rather fail on unknown control frames? + // But should we actually return false at any time? Yes? + break; + } + return true; +} + static bool ws_handler_on_frame (void *user_data, const struct ws_parser *parser) { struct ws_handler *self = user_data; - // TODO: handle pings and what not - // TODO: validate the message - // TODO: first concatenate all parts of the message - return self->on_message (self->user_data, + if (ws_is_control_frame (parser->opcode)) + return ws_handler_on_control_frame (self, parser); + + // TODO: do this rather in "on_frame_header" + if (self->message_data.len + parser->payload_len > self->max_payload_len) + { + ws_handler_fail (self, WS_STATUS_TOO_BIG); + return true; + } + + if (!self->expecting_continuation) + self->message_opcode = parser->opcode; + + str_append_data (&self->message_data, + parser->input.str, parser->payload_len); + self->expecting_continuation = !parser->is_fin; + + if (!parser->is_fin) + return true; + + bool result = self->on_message (self->user_data, self->message_opcode, self->parser.input.str, self->parser.payload_len); + str_reset (&self->message_data); + return result; +} + +static void +ws_handler_on_ping_timer (EV_P_ ev_timer *watcher, int revents) +{ + (void) loop; + (void) revents; + + struct ws_handler *self = watcher->data; + if (!self->received_pong) + { + // TODO: close/fail the connection? + return; + } + + ws_handler_send_control (self, WS_OPCODE_PING, NULL, 0); } static void @@ -1347,7 +1498,20 @@ ws_handler_init (struct ws_handler *self) str_init (&self->url); ws_parser_init (&self->parser); + self->parser.on_frame_header = ws_handler_on_frame_header; self->parser.on_frame = ws_handler_on_frame; + + str_init (&self->message_data); + + self->ping_interval = 60; + // This is still ridiculously high + self->max_payload_len = UINT32_MAX; + + // Just so we can safely stop it + ev_timer_init (&self->ping_timer, ws_handler_on_ping_timer, 0., 0.); + self->ping_timer.data = self; + // So that the first ping timer doesn't timeout the connection + self->received_pong = true; } static void @@ -1358,6 +1522,8 @@ ws_handler_free (struct ws_handler *self) str_map_free (&self->headers); str_free (&self->url); ws_parser_free (&self->parser); + str_free (&self->message_data); + ev_timer_stop (EV_DEFAULT_ &self->ping_timer); } static void @@ -1395,10 +1561,11 @@ ws_handler_on_header_value (http_parser *parser, const char *at, size_t len) static int ws_handler_on_headers_complete (http_parser *parser) { - // Just return 1 to tell the parser we don't want to parse any body; - // the parser should have found an upgrade request for WebSockets - (void) parser; - return 1; + // We strictly require a protocol upgrade + if (!parser->upgrade) + return 2; + + return 0; } static int @@ -1418,6 +1585,7 @@ ws_handler_finish_handshake (struct ws_handler *self) || self->hp.http_major != 1 || self->hp.http_minor != 1) ; // TODO: error (maybe send a frame depending on conditions) + // ...mostly just 400 Bad Request const char *upgrade = str_map_find (&self->headers, "Upgrade"); @@ -1425,6 +1593,12 @@ ws_handler_finish_handshake (struct ws_handler *self) const char *version = str_map_find (&self->headers, SEC_WS_VERSION); const char *protocol = str_map_find (&self->headers, SEC_WS_PROTOCOL); + if (!upgrade || strcmp (upgrade, "websocket") + || !version || strcmp (version, "13")) + ; // TODO: error + // ... if the version doesn't match, we must send back a header indicating + // the version we do support + struct str response; str_init (&response); str_append (&response, "HTTP/1.1 101 Switching Protocols\r\n"); @@ -1433,8 +1607,7 @@ ws_handler_finish_handshake (struct ws_handler *self) // TODO: prepare the rest of the headers - // TODO: we should ideally check that this is a 16-byte base64-encoded - // value; do we also have to strip surrounding whitespace? + // TODO: we should ideally check that this is a 16-byte base64-encoded value char *response_key = ws_encode_response_key (key); str_append_printf (&response, SEC_WS_ACCEPT ": %s\r\n", response_key); free (response_key); @@ -1442,6 +1615,10 @@ ws_handler_finish_handshake (struct ws_handler *self) str_append (&response, "\r\n"); self->write_cb (self->user_data, response.str, response.len); str_free (&response); + + // XXX: maybe we should start it earlier so that the handshake can + // timeout as well. ws_handler_connected()? + ev_timer_start (EV_DEFAULT_ &self->ping_timer); return true; } @@ -1477,14 +1654,16 @@ ws_handler_push (struct ws_handler *self, const void *data, size_t len) self->state = WS_HANDLER_WEBSOCKETS; return true; } - else if (n_parsed != len || HTTP_PARSER_ERRNO (&self->hp) != HPE_OK) + + if (n_parsed != len || HTTP_PARSER_ERRNO (&self->hp) != HPE_OK) { // TODO: error // print_debug (..., http_errno_description // (HTTP_PARSER_ERRNO (&self->hp)); + // NOTE: if == HPE_CB_headers_complete, "Upgrade" is missing + return false; } - // TODO: make double sure to handle the case of !upgrade return true; } @@ -1497,6 +1676,7 @@ static struct config_item g_config_table[] = { "port_scgi", NULL, "Port to bind for SCGI" }, { "port_ws", NULL, "Port to bind for WebSockets" }, { "pid_file", NULL, "Full path for the PID file" }, + // XXX: here belongs something like a web SPA that interfaces with us { "static_root", NULL, "The root for static content" }, { NULL, NULL, NULL } }; @@ -1526,11 +1706,16 @@ server_context_init (struct server_context *self) load_config_defaults (&self->config, g_config_table); } +static void close_listeners (struct server_context *self); + static void server_context_free (struct server_context *self) { - // TODO: free the clients (?) - // TODO: close the listeners (?) + // We really shouldn't attempt a quit without closing the clients first + soft_assert (!self->clients); + + close_listeners (self); + free (self->listeners); str_map_free (&self->config); } @@ -1773,7 +1958,8 @@ struct request { struct server_context *ctx; ///< Server context - void *user_data; ///< User data argument for callbacks + struct request_handler *handler; ///< Current request handler + void *handler_data; ///< User data for the handler /// Callback to write some CGI response data to the output void (*write_cb) (void *user_data, const void *data, size_t len); @@ -1782,16 +1968,17 @@ struct request /// CALLING THIS MAY CAUSE THE REQUEST TO BE DESTROYED. void (*close_cb) (void *user_data); - struct request_handler *handler; ///< Current request handler - void *handler_data; ///< User data for the handler + void *user_data; ///< User data argument for callbacks }; struct request_handler { LIST_HEADER (struct request_handler) - /// Install ourselves as the handler for the request if applicable - bool (*try_handle) (struct request *request, struct str_map *headers); + /// Install ourselves as the handler for the request if applicable. + /// Set @a continue_ to false if further processing should be stopped. + bool (*try_handle) (struct request *request, + struct str_map *headers, bool *continue_); /// Handle incoming data. /// Return false if further processing should be stopped. @@ -1826,16 +2013,22 @@ request_finish (struct request *self) static bool request_start (struct request *self, struct str_map *headers) { - LIST_FOR_EACH (struct request_handler, handler, self->ctx->handlers) - if (handler->try_handle (self, headers)) - { - // XXX: maybe we should isolate the handlers a bit more - self->handler = handler; + // XXX: it feels like this should rather be two steps: + // bool (*can_handle) (request *, headers) + // ... install the handler ... + // bool (*handle) (request *) + // + // However that might cause some stuff to be done twice. + // + // Another way we could get rid off the continue_ argument is via adding + // some way of marking the request as finished from within the handler. - // TODO: we should also allow the "try_handle" function to - // return that it has already finished processing the request - // and we should abort it by returning false here. - return true; + bool continue_ = true; + LIST_FOR_EACH (struct request_handler, handler, self->ctx->handlers) + if (handler->try_handle (self, headers, &continue_)) + { + self->handler = handler; + return continue_; } // Unable to serve the request @@ -1862,7 +2055,7 @@ request_push (struct request *self, const void *data, size_t len) static bool request_handler_json_rpc_try_handle - (struct request *request, struct str_map *headers) + (struct request *request, struct str_map *headers, bool *continue_) { const char *content_type = str_map_find (headers, "CONTENT_TYPE"); const char *method = str_map_find (headers, "REQUEST_METHOD"); @@ -1875,6 +2068,7 @@ request_handler_json_rpc_try_handle str_init (buf); request->handler_data = buf; + *continue_ = true; return true; } @@ -1972,8 +2166,11 @@ detect_magic (const void *data, size_t len) static bool request_handler_static_try_handle - (struct request *request, struct str_map *headers) + (struct request *request, struct str_map *headers, bool *continue_) { + // Serving static files is actually quite complicated as it turns out; + // but this is only meant to serve a few tiny text files + struct server_context *ctx = request->ctx; const char *root = str_map_find (&ctx->config, "static_root"); if (!root) @@ -1999,6 +2196,7 @@ request_handler_static_try_handle char *suffix = canonicalize_url_path (path_info); char *path = xstrdup_printf ("%s%s", root, suffix); + // TODO: check that this is a regular file FILE *fp = fopen (path, "rb"); if (!fp) { @@ -2045,6 +2243,13 @@ request_handler_static_try_handle while ((len = fread (buf, 1, sizeof buf, fp))) request->write_cb (request->user_data, buf, len); fclose (fp); + + // TODO: this should rather not be returned all at once but in chunks; + // file read requests never return EAGAIN + // TODO: actual file data should really be returned by a callback when + // the socket is writable with nothing to be sent (pumping the entire + // file all at once won't really work if it's huge). + *continue_ = false; return true; } @@ -2178,7 +2383,8 @@ static void client_fcgi_request_close (void *user_data) { struct client_fcgi_request *request = user_data; - // TODO: tell the fcgi_request to what? + // TODO: fcgi_request_finish()? That will most probably end up with us + // receiving client_fcgi_request_destroy() } static void * @@ -2186,6 +2392,7 @@ client_fcgi_request_start (void *user_data, struct fcgi_request *fcgi_request) { struct client *client = user_data; + // TODO: what if the request is aborted by ; struct client_fcgi_request *request = xmalloc (sizeof *request); request->fcgi_request = fcgi_request; request_init (&request->request); @@ -2375,12 +2582,17 @@ client_ws_write (void *user_data, const void *data, size_t len) } static bool -client_ws_on_message (void *user_data, const void *data, size_t len) +client_ws_on_message (void *user_data, + enum ws_opcode type, const void *data, size_t len) { struct client *client = user_data; struct client_ws *self = client->impl_data; - // TODO: do something about the message + struct str response; + str_init (&response); + process_json_rpc (client->ctx, data, len, &response); + // TODO: send the response + str_free (&response); return true; } @@ -2431,6 +2643,22 @@ struct listener struct client_impl *impl; ///< Client behaviour }; +static void +close_listeners (struct server_context *self) +{ + // TODO: factor out the closing act, to be used in initiate_quit() + for (size_t i = 0; i < self->n_listeners; i++) + { + struct listener *listener = &self->listeners[i]; + if (listener->fd == -1) + continue; + + ev_io_stop (EV_DEFAULT_ &listener->watcher); + xclose (listener->fd); + listener->fd = -1; + } +} + static bool client_read_loop (EV_P_ struct client *client, ev_io *watcher) { @@ -2472,6 +2700,8 @@ on_client_ready (EV_P_ ev_io *watcher, int revents) // finished flushing the write queue? This should probably even be // the default behaviour, as it's fairly uncommon for clients to // shutdown the socket for writes while leaving it open for reading. + // TODO: some sort of "on_buffers_flushed" callback for streaming huge + // chunks of external (or generated) data. if (!flush_queue (&client->write_queue, watcher)) goto close; return; From c87d684154875b3711168680c82e5b3b358dbdf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Sun, 15 Mar 2015 04:32:04 +0100 Subject: [PATCH 12/79] Steady progress Started parsing Content-Type properly after studying the HTTP RFC for a significant period of time. Some further WebSockets stuff. --- CMakeLists.txt | 3 +- demo-json-rpc-server.c | 494 +++++++++++++++++++++++++++++++++-------- 2 files changed, 407 insertions(+), 90 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c9c5762..97c63cb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,7 +36,8 @@ configure_file (${PROJECT_SOURCE_DIR}/config.h.in ${PROJECT_BINARY_DIR}/config.h include_directories (${PROJECT_BINARY_DIR}) # Build the executables -add_executable (demo-json-rpc-server demo-json-rpc-server.c) +add_executable (demo-json-rpc-server + demo-json-rpc-server.c http-parser/http_parser.c) target_link_libraries (demo-json-rpc-server ${project_libraries}) # The files to be installed diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index 77c9ca5..0cae4a0 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -38,9 +38,7 @@ #include #include -// FIXME: don't include the implementation, include the header and compile -// the implementation separately -#include "http-parser/http_parser.c" +#include "http-parser/http_parser.h" // --- Extensions to liberty --------------------------------------------------- @@ -129,6 +127,17 @@ tolower_ascii_strxfrm (char *dest, const char *src, size_t n) return len; } +static int +strcasecmp_ascii (const char *a, const char *b) +{ + while (*a && *b) + if (tolower_ascii (*a) != tolower_ascii (*b)) + break; + return *a - *b; +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + static void base64_encode (const void *data, size_t len, struct str *output) { @@ -169,6 +178,211 @@ base64_encode (const void *data, size_t len, struct str *output) } } +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +// Basic tokenizer for HTTP headers, to be used in various parsers. +// The input should already be unwrapped. + +enum http_tokenizer_field +{ + HTTP_T_EOF, ///< Input error + HTTP_T_ERROR, ///< End of input + + HTTP_T_TOKEN, ///< "token" + HTTP_T_QUOTED_STRING, ///< "quoted-string" + HTTP_T_SEPARATOR ///< "separators" +}; + +struct http_tokenizer +{ + const char *input; ///< The input string + size_t input_len; ///< Length of the input + size_t offset; ///< Position in the input + + char separator; ///< The separator character + struct str string; ///< "token" / "quoted-string" content +}; + +static void +http_tokenizer_init (struct http_tokenizer *self, const char *input) +{ + memset (self, 0, sizeof *self); + self->input = input; + self->input_len = strlen (input); + + str_init (&self->string); +} + +static void +http_tokenizer_free (struct http_tokenizer *self) +{ + str_free (&self->string); +} + +static bool +http_tokenizer_is_ctl (int c) +{ + return (c >= 0 && c <= 31) || c == 127; +} + +static bool +http_tokenizer_is_char (int c) +{ + return c >= 0 && c <= 127; +} + +static enum http_tokenizer_field +http_tokenizer_quoted_string (struct http_tokenizer *self) +{ + bool quoted_pair = false; + while (self->offset < self->input_len) + { + int c = self->input[self->offset++]; + if (quoted_pair) + { + if (!http_tokenizer_is_char (c)) + return HTTP_T_ERROR; + + str_append_c (&self->string, c); + quoted_pair = false; + } + else if (c == '\\') + quoted_pair = true; + else if (c == '"') + return HTTP_T_QUOTED_STRING; + else if (http_tokenizer_is_ctl (c)) + return HTTP_T_ERROR; + else + str_append_c (&self->string, c); + } + + // Premature end of input + return HTTP_T_ERROR; +} + +static enum http_tokenizer_field +http_tokenizer_next (struct http_tokenizer *self, bool skip_lws) +{ + const char *separators = "()<>@.;:\\\"/[]?={} \t"; + + str_reset (&self->string); + if (self->offset >= self->input_len) + return HTTP_T_EOF; + + int c = self->input[self->offset++]; + + if (skip_lws) + while (c == ' ' || c == '\t') + { + if (self->offset >= self->input_len) + return HTTP_T_EOF; + c = self->input[self->offset++]; + } + + if (c == '"') + return http_tokenizer_quoted_string (self); + + if (strchr (separators, c)) + { + self->separator = c; + return HTTP_T_SEPARATOR; + } + + if (!http_tokenizer_is_char (c) + || http_tokenizer_is_ctl (c)) + return HTTP_T_ERROR; + + str_append_c (&self->string, c); + while (self->offset < self->input_len) + { + c = self->input[self->offset]; + if (!http_tokenizer_is_char (c) + || http_tokenizer_is_ctl (c) + || strchr (separators, c)) + break; + + str_append_c (&self->string, c); + self->offset++; + } + return HTTP_T_TOKEN; +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +static bool +http_parse_media_type_parameter + (struct http_tokenizer *t, struct str_map *parameters) +{ + bool result = false; + char *attribute = NULL; + + if (http_tokenizer_next (t, true) != HTTP_T_TOKEN) + goto end; + attribute = xstrdup (t->string.str); + + if (http_tokenizer_next (t, false) != HTTP_T_SEPARATOR + || t->separator != '=') + goto end; + + switch (http_tokenizer_next (t, false)) + { + case HTTP_T_TOKEN: + case HTTP_T_QUOTED_STRING: + str_map_set (parameters, attribute, xstrdup (t->string.str)); + result = true; + default: + break; + } + +end: + free (attribute); + return result; +} + +/// Parser for Accept and Content-Type. @a type and @a subtype may be non-NULL +/// even if the function fails. @a parameters should be case-insensitive. +static bool +http_parse_media_type (const char *media_type, + char **type, char **subtype, struct str_map *parameters) +{ + // The parsing is strict wrt. LWS as per RFC 2616 section 3.7 + + bool result = false; + struct http_tokenizer t; + http_tokenizer_init (&t, media_type); + + if (http_tokenizer_next (&t, true) != HTTP_T_TOKEN) + goto end; + *type = xstrdup (t.string.str); + + if (http_tokenizer_next (&t, false) != HTTP_T_SEPARATOR + || t.separator != '/') + goto end; + + if (http_tokenizer_next (&t, false) != HTTP_T_TOKEN) + goto end; + *subtype = xstrdup (t.string.str); + + while (true) + switch (http_tokenizer_next (&t, true)) + { + case HTTP_T_SEPARATOR: + if (t.separator != ';') + goto end; + if (!http_parse_media_type_parameter (&t, parameters)) + goto end; + break; + case HTTP_T_EOF: + result = true; + default: + goto end; + } + +end: + http_tokenizer_free (&t); + return result; +} + // --- libev helpers ----------------------------------------------------------- static bool @@ -1065,10 +1279,11 @@ scgi_parser_push (struct scgi_parser *self, #define WS_GUID "258EAFA5-E914-47DA-95CA-C5AB0DC85B11" -#define SEC_WS_KEY "Sec-WebSocket-Key" -#define SEC_WS_ACCEPT "Sec-WebSocket-Accept" -#define SEC_WS_PROTOCOL "Sec-WebSocket-Protocol" -#define SEC_WS_VERSION "Sec-WebSocket-Version" +#define SEC_WS_KEY "Sec-WebSocket-Key" +#define SEC_WS_ACCEPT "Sec-WebSocket-Accept" +#define SEC_WS_PROTOCOL "Sec-WebSocket-Protocol" +#define SEC_WS_EXTENSIONS "Sec-WebSocket-Extensions" +#define SEC_WS_VERSION "Sec-WebSocket-Version" #define WS_MAX_CONTROL_PAYLOAD_LEN 125 @@ -1290,10 +1505,6 @@ ws_parser_push (struct ws_parser *self, const void *data, size_t len) } } -// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -// TODO: something to build frames for data - // - - Server handler - - - - - - - - - - - - - - - - - - - - - - - - - - - - - // WebSockets aren't CGI-compatible, therefore we must handle the initial HTTP @@ -1304,7 +1515,8 @@ ws_parser_push (struct ws_parser *self, const void *data, size_t len) enum ws_handler_state { WS_HANDLER_HANDSHAKE, ///< Parsing HTTP - WS_HANDLER_WEBSOCKETS ///< Parsing WebSockets frames + WS_HANDLER_OPEN, ///< Parsing WebSockets frames + WS_HANDLER_CLOSING ///< Closing the connection }; struct ws_handler @@ -1327,7 +1539,7 @@ struct ws_handler unsigned ping_interval; ///< Ping interval in seconds uint64_t max_payload_len; ///< Maximum length of any message - // TODO: bool closing; // XXX: rather a { OPEN, CLOSING } state? + // TODO: handshake_timeout // TODO: a close timer // TODO: a ping timer (when no pong is received by the second time the @@ -1361,8 +1573,8 @@ struct ws_handler }; static void -ws_handler_send_control (struct ws_handler *self, enum ws_opcode opcode, - const void *data, size_t len) +ws_handler_send_control (struct ws_handler *self, + enum ws_opcode opcode, const void *data, size_t len) { if (len > WS_MAX_CONTROL_PAYLOAD_LEN) { @@ -1392,6 +1604,33 @@ ws_handler_fail (struct ws_handler *self, enum ws_status reason) // ignore frames up to a corresponding close from the client. // Read the RFC once again to see if we can really process the frames. +// TODO: add support for fragmented responses +static void +ws_handler_send (struct ws_handler *self, + enum ws_opcode opcode, const void *data, size_t len) +{ + struct str header; + str_init (&header); + str_pack_u8 (&header, 0x80 | (opcode & 0x0F)); + + if (len > UINT16_MAX) + { + str_pack_u8 (&header, 127); + str_pack_u64 (&header, len); + } + else if (len > 125) + { + str_pack_u8 (&header, 126); + str_pack_u16 (&header, len); + } + else + str_pack_u8 (&header, len); + + self->write_cb (self->user_data, header.str, header.len); + self->write_cb (self->user_data, data, len); + str_free (&header); +} + static bool ws_handler_on_frame_header (void *user_data, const struct ws_parser *parser) { @@ -1529,9 +1768,19 @@ ws_handler_free (struct ws_handler *self) static void ws_handler_on_header_read (struct ws_handler *self) { - // TODO: some headers can appear more than once, concatenate their values; - // for example "Sec-WebSocket-Version" - str_map_set (&self->headers, self->field.str, self->value.str); + const char *field = self->field.str; + bool can_concat = + !strcasecmp_ascii (field, SEC_WS_PROTOCOL) || + !strcasecmp_ascii (field, SEC_WS_EXTENSIONS); + + const char *current = str_map_find (&self->headers, field); + if (can_concat && current) + str_map_set (&self->headers, field, + xstrdup_printf ("%s, %s", current, self->value.str)); + else + // If the field cannot be concatenated, just overwrite the last value. + // Maybe we should issue a warning or something. + str_map_set (&self->headers, field, xstrdup (self->value.str)); } static int @@ -1576,48 +1825,104 @@ ws_handler_on_url (http_parser *parser, const char *at, size_t len) return 0; } +#define HTTP_101_SWITCHING_PROTOCOLS "101 Switching Protocols" +#define HTTP_400_BAD_REQUEST "400 Bad Request" +#define HTTP_405_METHOD_NOT_ALLOWED "405 Method Not Allowed" +#define HTTP_505_VERSION_NOT_SUPPORTED "505 HTTP Version Not Supported" + +static void +ws_handler_http_responsev (struct ws_handler *self, + const char *status, char *const *fields) +{ + hard_assert (status != NULL); + + struct str response; + str_init (&response); + str_append_printf (&response, "HTTP/1.1 %s\r\n", status); + + while (*fields) + str_append_printf (&response, "%s\r\n", *fields++); + + str_append (&response, "Server: " + PROGRAM_NAME "/" PROGRAM_VERSION "\r\n\r\n"); + self->write_cb (self->user_data, response.str, response.len); + str_free (&response); +} + +static void +ws_handler_http_response (struct ws_handler *self, const char *status, ...) +{ + struct str_vector v; + str_vector_init (&v); + + va_list ap; + va_start (ap, status); + + const char *s; + while ((s = va_arg (ap, const char *))) + str_vector_add (&v, s); + + va_end (ap); + + ws_handler_http_responsev (self, status, v.vector); + str_vector_free (&v); +} + +#define FAIL_HANDSHAKE(status, ...) \ + BLOCK_START \ + ws_handler_http_response (self, (status), __VA_ARGS__); \ + return false; \ + BLOCK_END + static bool ws_handler_finish_handshake (struct ws_handler *self) { - // TODO: probably factor this block out into its own function - // TODO: check if everything seems to be right - if (self->hp.method != HTTP_GET - || self->hp.http_major != 1 - || self->hp.http_minor != 1) - ; // TODO: error (maybe send a frame depending on conditions) - // ...mostly just 400 Bad Request + if (self->hp.http_major != 1 || self->hp.http_minor != 1) + FAIL_HANDSHAKE (HTTP_505_VERSION_NOT_SUPPORTED, NULL); + if (self->hp.method != HTTP_GET) + FAIL_HANDSHAKE (HTTP_405_METHOD_NOT_ALLOWED, "Allow: GET", NULL); + + // Reject weird URLs specifying the schema and the host + struct http_parser_url url; + if (http_parser_parse_url (self->url.str, self->url.len, false, &url) + || (url.field_set & (1 << UF_SCHEMA | 1 << UF_HOST | 1 << UF_PORT))) + FAIL_HANDSHAKE (HTTP_400_BAD_REQUEST, NULL); const char *upgrade = str_map_find (&self->headers, "Upgrade"); - + // TODO: we should ideally check that this is a 16-byte base64-encoded value const char *key = str_map_find (&self->headers, SEC_WS_KEY); const char *version = str_map_find (&self->headers, SEC_WS_VERSION); const char *protocol = str_map_find (&self->headers, SEC_WS_PROTOCOL); - if (!upgrade || strcmp (upgrade, "websocket") - || !version || strcmp (version, "13")) - ; // TODO: error - // ... if the version doesn't match, we must send back a header indicating - // the version we do support + if (!upgrade || strcmp (upgrade, "websocket") || !version) + FAIL_HANDSHAKE (HTTP_400_BAD_REQUEST, NULL); + if (strcmp (version, "13")) + FAIL_HANDSHAKE (HTTP_400_BAD_REQUEST, SEC_WS_VERSION ": 13", NULL); - struct str response; - str_init (&response); - str_append (&response, "HTTP/1.1 101 Switching Protocols\r\n"); - str_append (&response, "Upgrade: websocket\r\n"); - str_append (&response, "Connection: Upgrade\r\n"); + struct str_vector fields; + str_vector_init (&fields); - // TODO: prepare the rest of the headers + str_vector_add_args (&fields, + "Upgrade: websocket", + "Connection: Upgrade", + NULL); - // TODO: we should ideally check that this is a 16-byte base64-encoded value char *response_key = ws_encode_response_key (key); - str_append_printf (&response, SEC_WS_ACCEPT ": %s\r\n", response_key); + str_vector_add_owned (&fields, + xstrdup_printf (SEC_WS_ACCEPT ": %s", response_key)); free (response_key); - str_append (&response, "\r\n"); - self->write_cb (self->user_data, response.str, response.len); - str_free (&response); + // TODO: check and set Sec-Websocket-{Extensions,Protocol} + + ws_handler_http_responsev (self, + HTTP_101_SWITCHING_PROTOCOLS, fields.vector); + + str_vector_free (&fields); // XXX: maybe we should start it earlier so that the handshake can // timeout as well. ws_handler_connected()? + // + // But it should rather be named "connect_timer" ev_timer_start (EV_DEFAULT_ &self->ping_timer); return true; } @@ -1625,8 +1930,13 @@ ws_handler_finish_handshake (struct ws_handler *self) static bool ws_handler_push (struct ws_handler *self, const void *data, size_t len) { - if (self->state == WS_HANDLER_WEBSOCKETS) + if (self->state != WS_HANDLER_HANDSHAKE) + { + // TODO: handle the case of len == 0: + // OPEN: "on_close" WS_STATUS_ABNORMAL + // CLOSING: just close the connection return ws_parser_push (&self->parser, data, len); + } // The handshake hasn't been done yet, process HTTP headers static const http_parser_settings http_settings = @@ -1637,33 +1947,37 @@ ws_handler_push (struct ws_handler *self, const void *data, size_t len) .on_url = ws_handler_on_url, }; + // NOTE: the HTTP parser unfolds values and removes preceeding whitespace, + // but otherwise doesn't touch the values or the following whitespace; + // we might want to strip at least the trailing whitespace size_t n_parsed = http_parser_execute (&self->hp, &http_settings, data, len); if (self->hp.upgrade) { + // The handshake hasn't been finished, yet there is more data + // to be processed after the headers already if (len - n_parsed) - { - // TODO: error: the handshake hasn't been finished, yet there - // is more data to process after the headers - } + FAIL_HANDSHAKE (HTTP_400_BAD_REQUEST, NULL); if (!ws_handler_finish_handshake (self)) return false; - self->state = WS_HANDLER_WEBSOCKETS; + self->state = WS_HANDLER_OPEN; return true; } - if (n_parsed != len || HTTP_PARSER_ERRNO (&self->hp) != HPE_OK) + enum http_errno err = HTTP_PARSER_ERRNO (&self->hp); + if (n_parsed != len || err != HPE_OK) { - // TODO: error - // print_debug (..., http_errno_description - // (HTTP_PARSER_ERRNO (&self->hp)); - // NOTE: if == HPE_CB_headers_complete, "Upgrade" is missing - return false; - } + if (err == HPE_CB_headers_complete) + print_debug ("WS handshake failed: %s", "missing `Upgrade' field"); + else + print_debug ("WS handshake failed: %s", + http_errno_description (err)); + FAIL_HANDSHAKE (HTTP_400_BAD_REQUEST, NULL); + } return true; } @@ -1776,45 +2090,37 @@ json_rpc_response (json_t *id, json_t *result, json_t *error) // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - static bool -try_advance (const char **p, const char *text) +validate_json_rpc_content_type (const char *content_type) { - size_t len = strlen (text); - if (strncmp (*p, text, len)) - return false; + char *type = NULL; + char *subtype = NULL; - *p += len; - return true; -} + struct str_map parameters; + str_map_init (¶meters); + parameters.free = free; + parameters.key_xfrm = tolower_ascii_strxfrm; -static bool -validate_json_rpc_content_type (const char *type) -{ - const char *content_types[] = - { - "application/json-rpc", // obsolete - "application/json" - }; - const char *tails[] = - { - "; charset=utf-8", - "; charset=UTF-8", - "" - }; + bool result = http_parse_media_type + (content_type, &type, &subtype, ¶meters); + if (!result) + goto end; - bool found = false; - for (size_t i = 0; i < N_ELEMENTS (content_types); i++) - if ((found = try_advance (&type, content_types[i]))) - break; - if (!found) - return false; + if (strcasecmp_ascii (type, "application") + || (strcasecmp_ascii (subtype, "json") && + strcasecmp_ascii (subtype, "json-rpc" /* obsolete */))) + result = false; - for (size_t i = 0; i < N_ELEMENTS (tails); i++) - if ((found = try_advance (&type, tails[i]))) - break; - if (!found) - return false; + const char *charset = str_map_find (¶meters, "charset"); + if (charset && strcasecmp_ascii (charset, "UTF-8")) + result = false; - return !*type; + // Currently ignoring all unknown parametrs + +end: + free (type); + free (subtype); + str_map_free (¶meters); + return result; } // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -2588,10 +2894,17 @@ client_ws_on_message (void *user_data, struct client *client = user_data; struct client_ws *self = client->impl_data; + if (type != WS_OPCODE_TEXT) + { + ws_handler_fail (&self->handler, WS_STATUS_UNACCEPTABLE); + return false; + } + struct str response; str_init (&response); process_json_rpc (client->ctx, data, len, &response); - // TODO: send the response + ws_handler_send (&self->handler, + WS_OPCODE_TEXT, response.str, response.len); str_free (&response); return true; } @@ -2607,6 +2920,9 @@ client_ws_init (struct client *client) self->handler.on_message = client_ws_on_message; self->handler.user_data = client; // TODO: configure the handler some more, e.g. regarding the protocol + + // One mebibyte seems to be a reasonable value + self->handler.max_payload_len = 1 << 10; } static void From 9b7dd630e3d91e21dfc7b077a4414b127fed94e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Sun, 22 Mar 2015 22:35:58 +0100 Subject: [PATCH 13/79] WebSockets improvements - validate more HTTP stuff, use the newer RFC - validate the base64 key --- demo-json-rpc-server.c | 418 ++++++++++++++++++++++++++++++++++------- 1 file changed, 352 insertions(+), 66 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index 0cae4a0..06f0fc4 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -136,8 +136,93 @@ strcasecmp_ascii (const char *a, const char *b) return *a - *b; } +static bool +isspace_ascii (int c) +{ + return c == '\f' || c == '\n' || c == '\r' || c == '\t' || c == '\v'; +} + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +static uint8_t g_base64_table[256] = +{ + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 62, 64, 64, 64, 63, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 64, 64, 64, 0, 64, 64, + 64, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 64, 64, 64, 64, 64, + 64, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 64, 64, 64, 64, 64, + + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, +}; + +static inline bool +base64_decode_group (const char **s, struct str *output) +{ + uint8_t input[4]; + size_t loaded = 0; + for (; loaded < 4; (*s)++) + { + if (!**s) + return loaded == 0; + if (!isspace_ascii (**s)) + input[loaded++] = **s; + } + + size_t len = 3; + if (input[0] == '=' || input[1] == '=') + return false; + if (input[2] == '=' && input[3] != '=') + return false; + if (input[2] == '=') + len--; + if (input[3] == '=') + len--; + + uint8_t a = g_base64_table[input[0]]; + uint8_t b = g_base64_table[input[1]]; + uint8_t c = g_base64_table[input[2]]; + uint8_t d = g_base64_table[input[3]]; + + if (((a | b) | (c | d)) & 0x40) + return false; + + uint32_t block = a << 18 | b << 12 | c << 6 | d; + switch (len) + { + case 1: + str_append_c (output, block >> 16); + break; + case 2: + str_append_c (output, block >> 16); + str_append_c (output, block >> 8); + break; + case 3: + str_append_c (output, block >> 16); + str_append_c (output, block >> 8); + str_append_c (output, block); + } + return true; +} + +static bool +base64_decode (const char *s, struct str *output) +{ + while (*s) + if (!base64_decode_group (&s, output)) + return false; + return true; +} + static void base64_encode (const void *data, size_t len, struct str *output) { @@ -178,37 +263,73 @@ base64_encode (const void *data, size_t len, struct str *output) } } -// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// --- HTTP parsing ------------------------------------------------------------ -// Basic tokenizer for HTTP headers, to be used in various parsers. +// Basic tokenizer for HTTP header field values, to be used in various parsers. // The input should already be unwrapped. -enum http_tokenizer_field +// Recommended literature: +// http://tools.ietf.org/html/rfc7230#section-3.2.6 +// http://tools.ietf.org/html/rfc7230#appendix-B +// http://tools.ietf.org/html/rfc5234#appendix-B.1 + +#define HTTP_TOKENIZER_CLASS(name, definition) \ + static inline bool \ + http_tokenizer_is_ ## name (int c) \ + { \ + return (definition); \ + } + +HTTP_TOKENIZER_CLASS (vchar, c >= 0x21 && c <= 0x7E) +HTTP_TOKENIZER_CLASS (delimiter, !!strchr ("\"(),/:;<=>?@[\\]{}", c)) +HTTP_TOKENIZER_CLASS (whitespace, c == '\t' || c == ' ') +HTTP_TOKENIZER_CLASS (obs_text, c >= 0x80 && c <= 0xFF) + +HTTP_TOKENIZER_CLASS (tchar, + http_tokenizer_is_vchar (c) && !http_tokenizer_is_delimiter (c)) + +HTTP_TOKENIZER_CLASS (qdtext, + c == '\t' || c == ' ' || c == '!' + || (c >= 0x23 && c <= 0x5B) + || (c >= 0x5D && c <= 0x7E) + || http_tokenizer_is_obs_text (c)) + +HTTP_TOKENIZER_CLASS (quoted_pair, + c == '\t' || c == ' ' + || http_tokenizer_is_vchar (c) + || http_tokenizer_is_obs_text (c)) + +#undef HTTP_TOKENIZER_CLASS + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +enum http_tokenizer_token { HTTP_T_EOF, ///< Input error HTTP_T_ERROR, ///< End of input HTTP_T_TOKEN, ///< "token" HTTP_T_QUOTED_STRING, ///< "quoted-string" - HTTP_T_SEPARATOR ///< "separators" + HTTP_T_DELIMITER, ///< "delimiters" + HTTP_T_WHITESPACE ///< RWS/OWS/BWS }; struct http_tokenizer { - const char *input; ///< The input string + const unsigned char *input; ///< The input string size_t input_len; ///< Length of the input size_t offset; ///< Position in the input - char separator; ///< The separator character + char delimiter; ///< The delimiter character struct str string; ///< "token" / "quoted-string" content }; static void -http_tokenizer_init (struct http_tokenizer *self, const char *input) +http_tokenizer_init (struct http_tokenizer *self, const char *input, size_t len) { memset (self, 0, sizeof *self); - self->input = input; - self->input_len = strlen (input); + self->input = (const unsigned char *) input; + self->input_len = len; str_init (&self->string); } @@ -219,19 +340,7 @@ http_tokenizer_free (struct http_tokenizer *self) str_free (&self->string); } -static bool -http_tokenizer_is_ctl (int c) -{ - return (c >= 0 && c <= 31) || c == 127; -} - -static bool -http_tokenizer_is_char (int c) -{ - return c >= 0 && c <= 127; -} - -static enum http_tokenizer_field +static enum http_tokenizer_token http_tokenizer_quoted_string (struct http_tokenizer *self) { bool quoted_pair = false; @@ -240,7 +349,7 @@ http_tokenizer_quoted_string (struct http_tokenizer *self) int c = self->input[self->offset++]; if (quoted_pair) { - if (!http_tokenizer_is_char (c)) + if (!http_tokenizer_is_quoted_pair (c)) return HTTP_T_ERROR; str_append_c (&self->string, c); @@ -250,29 +359,27 @@ http_tokenizer_quoted_string (struct http_tokenizer *self) quoted_pair = true; else if (c == '"') return HTTP_T_QUOTED_STRING; - else if (http_tokenizer_is_ctl (c)) - return HTTP_T_ERROR; - else + else if (http_tokenizer_is_qdtext (c)) str_append_c (&self->string, c); + else + return HTTP_T_ERROR; } // Premature end of input return HTTP_T_ERROR; } -static enum http_tokenizer_field -http_tokenizer_next (struct http_tokenizer *self, bool skip_lws) +static enum http_tokenizer_token +http_tokenizer_next (struct http_tokenizer *self, bool skip_ows) { - const char *separators = "()<>@.;:\\\"/[]?={} \t"; - str_reset (&self->string); if (self->offset >= self->input_len) return HTTP_T_EOF; int c = self->input[self->offset++]; - if (skip_lws) - while (c == ' ' || c == '\t') + if (skip_ows) + while (http_tokenizer_is_whitespace (c)) { if (self->offset >= self->input_len) return HTTP_T_EOF; @@ -282,29 +389,38 @@ http_tokenizer_next (struct http_tokenizer *self, bool skip_lws) if (c == '"') return http_tokenizer_quoted_string (self); - if (strchr (separators, c)) + if (http_tokenizer_is_delimiter (c)) { - self->separator = c; - return HTTP_T_SEPARATOR; + self->delimiter = c; + return HTTP_T_DELIMITER; } - if (!http_tokenizer_is_char (c) - || http_tokenizer_is_ctl (c)) + // Simple variable-length tokens + enum http_tokenizer_token result; + bool (*eater) (int c) = NULL; + if (http_tokenizer_is_whitespace (c)) + { + eater = http_tokenizer_is_whitespace; + result = HTTP_T_WHITESPACE; + } + else if (http_tokenizer_is_tchar (c)) + { + eater = http_tokenizer_is_tchar; + result = HTTP_T_TOKEN; + } + else return HTTP_T_ERROR; str_append_c (&self->string, c); while (self->offset < self->input_len) { - c = self->input[self->offset]; - if (!http_tokenizer_is_char (c) - || http_tokenizer_is_ctl (c) - || strchr (separators, c)) + if (!eater (c = self->input[self->offset])) break; str_append_c (&self->string, c); self->offset++; } - return HTTP_T_TOKEN; + return result; } // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -320,8 +436,8 @@ http_parse_media_type_parameter goto end; attribute = xstrdup (t->string.str); - if (http_tokenizer_next (t, false) != HTTP_T_SEPARATOR - || t->separator != '=') + if (http_tokenizer_next (t, false) != HTTP_T_DELIMITER + || t->delimiter != '=') goto end; switch (http_tokenizer_next (t, false)) @@ -339,24 +455,22 @@ end: return result; } -/// Parser for Accept and Content-Type. @a type and @a subtype may be non-NULL +/// Parser for "Content-Type". @a type and @a subtype may be non-NULL /// even if the function fails. @a parameters should be case-insensitive. static bool http_parse_media_type (const char *media_type, char **type, char **subtype, struct str_map *parameters) { - // The parsing is strict wrt. LWS as per RFC 2616 section 3.7 - bool result = false; struct http_tokenizer t; - http_tokenizer_init (&t, media_type); + http_tokenizer_init (&t, media_type, strlen (media_type)); if (http_tokenizer_next (&t, true) != HTTP_T_TOKEN) goto end; *type = xstrdup (t.string.str); - if (http_tokenizer_next (&t, false) != HTTP_T_SEPARATOR - || t.separator != '/') + if (http_tokenizer_next (&t, false) != HTTP_T_DELIMITER + || t.delimiter != '/') goto end; if (http_tokenizer_next (&t, false) != HTTP_T_TOKEN) @@ -366,8 +480,8 @@ http_parse_media_type (const char *media_type, while (true) switch (http_tokenizer_next (&t, true)) { - case HTTP_T_SEPARATOR: - if (t.separator != ';') + case HTTP_T_DELIMITER: + if (t.delimiter != ';') goto end; if (!http_parse_media_type_parameter (&t, parameters)) goto end; @@ -383,6 +497,125 @@ end: return result; } +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +struct http_protocol +{ + LIST_HEADER (struct http_protocol) + + char *name; ///< The protocol to upgrade to + char *version; ///< Version of the protocol, if any +}; + +static void +http_protocol_destroy (struct http_protocol *self) +{ + free (self->name); + free (self->version); + free (self); +} + +static bool +http_parse_upgrade (const char *upgrade, struct http_protocol **out) +{ + // HTTP grammar makes this more complicated than it should be + + bool result = false; + struct http_protocol *list = NULL; + struct http_protocol *tail = NULL; + + struct http_tokenizer t; + http_tokenizer_init (&t, upgrade, strlen (upgrade)); + + enum { + STATE_PROTOCOL_NAME, + STATE_SLASH, + STATE_PROTOCOL_VERSION, + STATE_EXPECT_COMMA + } state = STATE_PROTOCOL_NAME; + struct http_protocol *proto = NULL; + + while (true) + switch (state) + { + case STATE_PROTOCOL_NAME: + switch (http_tokenizer_next (&t, false)) + { + case HTTP_T_DELIMITER: + if (t.delimiter != ',') + goto end; + case HTTP_T_WHITESPACE: + break; + case HTTP_T_TOKEN: + proto = xcalloc (1, sizeof *proto); + proto->name = xstrdup (t.string.str); + LIST_APPEND_WITH_TAIL (list, tail, proto); + state = STATE_SLASH; + break; + case HTTP_T_EOF: + result = true; + default: + goto end; + } + break; + case STATE_SLASH: + switch (http_tokenizer_next (&t, false)) + { + case HTTP_T_DELIMITER: + if (t.delimiter == '/') + state = STATE_PROTOCOL_VERSION; + else if (t.delimiter == ',') + state = STATE_PROTOCOL_NAME; + else + goto end; + break; + case HTTP_T_WHITESPACE: + state = STATE_EXPECT_COMMA; + break; + case HTTP_T_EOF: + result = true; + default: + goto end; + } + break; + case STATE_PROTOCOL_VERSION: + switch (http_tokenizer_next (&t, false)) + { + case HTTP_T_TOKEN: + proto->version = xstrdup (t.string.str); + state = STATE_EXPECT_COMMA; + break; + default: + goto end; + } + break; + case STATE_EXPECT_COMMA: + switch (http_tokenizer_next (&t, false)) + { + case HTTP_T_DELIMITER: + if (t.delimiter != ',') + goto end; + state = STATE_PROTOCOL_NAME; + case HTTP_T_WHITESPACE: + break; + case HTTP_T_EOF: + result = true; + default: + goto end; + } + } + +end: + if (result) + *out = list; + else + LIST_FOR_EACH (struct http_protocol, iter, list) + http_protocol_destroy (iter); + + http_tokenizer_free (&t); + return result; +} + // --- libev helpers ----------------------------------------------------------- static bool @@ -1765,16 +1998,38 @@ ws_handler_free (struct ws_handler *self) ev_timer_stop (EV_DEFAULT_ &self->ping_timer); } +static bool +ws_handler_header_field_is_a_list (const char *name) +{ + // This must contain all header fields we use for anything + static const char *concatenable[] = + { SEC_WS_PROTOCOL, SEC_WS_EXTENSIONS, "Connection", "Upgrade" }; + + for (size_t i = 0; i < N_ELEMENTS (concatenable); i++) + if (!strcasecmp_ascii (name, concatenable[i])) + return true; + return false; +} + static void ws_handler_on_header_read (struct ws_handler *self) { - const char *field = self->field.str; - bool can_concat = - !strcasecmp_ascii (field, SEC_WS_PROTOCOL) || - !strcasecmp_ascii (field, SEC_WS_EXTENSIONS); + // The HTTP parser unfolds values and removes preceding whitespace, but + // otherwise doesn't touch the values or the following whitespace. + // RFC 7230 states that trailing whitespace is not part of a field value + char *value = self->field.str; + size_t len = self->field.len; + while (len--) + if (value[len] == '\t' || value[len] == ' ') + value[len] = '\0'; + else + break; + self->field.len = len; + + const char *field = self->field.str; const char *current = str_map_find (&self->headers, field); - if (can_concat && current) + if (ws_handler_header_field_is_a_list (field) && current) str_map_set (&self->headers, field, xstrdup_printf ("%s, %s", current, self->value.str)); else @@ -1828,6 +2083,7 @@ ws_handler_on_url (http_parser *parser, const char *at, size_t len) #define HTTP_101_SWITCHING_PROTOCOLS "101 Switching Protocols" #define HTTP_400_BAD_REQUEST "400 Bad Request" #define HTTP_405_METHOD_NOT_ALLOWED "405 Method Not Allowed" +#define HTTP_417_EXPECTATION_FAILED "407 Expectation Failed" #define HTTP_505_VERSION_NOT_SUPPORTED "505 HTTP Version Not Supported" static void @@ -1877,24 +2133,57 @@ ws_handler_http_response (struct ws_handler *self, const char *status, ...) static bool ws_handler_finish_handshake (struct ws_handler *self) { - if (self->hp.http_major != 1 || self->hp.http_minor != 1) + // XXX: we probably shouldn't use 505 to reject the minor version but w/e + if (self->hp.http_major != 1 || self->hp.http_minor < 1) FAIL_HANDSHAKE (HTTP_505_VERSION_NOT_SUPPORTED, NULL); if (self->hp.method != HTTP_GET) FAIL_HANDSHAKE (HTTP_405_METHOD_NOT_ALLOWED, "Allow: GET", NULL); - // Reject weird URLs specifying the schema and the host + // Your expectations are way too high + if (str_map_find (&self->headers, "Expect")) + FAIL_HANDSHAKE (HTTP_417_EXPECTATION_FAILED, NULL); + + // Reject URLs specifying the schema and host; we're not parsing that + // TODO: actually do parse this and let our user decide if it matches struct http_parser_url url; if (http_parser_parse_url (self->url.str, self->url.len, false, &url) - || (url.field_set & (1 << UF_SCHEMA | 1 << UF_HOST | 1 << UF_PORT))) + || (url.field_set & (1 << UF_SCHEMA | 1 << UF_HOST | 1 << UF_PORT)) + || !str_map_find (&self->headers, "Host")) FAIL_HANDSHAKE (HTTP_400_BAD_REQUEST, NULL); - const char *upgrade = str_map_find (&self->headers, "Upgrade"); - // TODO: we should ideally check that this is a 16-byte base64-encoded value + const char *connection = str_map_find (&self->headers, "Connection"); + if (!connection || strcasecmp_ascii (connection, "Upgrade")) + FAIL_HANDSHAKE (HTTP_400_BAD_REQUEST, NULL); + + // Check if we can actually upgrade the protocol to WebSockets + const char *upgrade = str_map_find (&self->headers, "Upgrade"); + struct http_protocol *offered_upgrades = NULL; + bool can_upgrade = false; + if (upgrade && http_parse_upgrade (upgrade, &offered_upgrades)) + // Case-insensitive according to RFC 6455; neither RFC 2616 nor 7230 + // say anything at all about case-sensitivity for this field + LIST_FOR_EACH (struct http_protocol, iter, offered_upgrades) + { + if (!iter->version && !strcasecmp_ascii (iter->name, "websocket")) + can_upgrade = true; + http_protocol_destroy (iter); + } + if (!can_upgrade) + FAIL_HANDSHAKE (HTTP_400_BAD_REQUEST, NULL); + + // Okay, we've finally got past basic HTTP/1.1 stuff const char *key = str_map_find (&self->headers, SEC_WS_KEY); const char *version = str_map_find (&self->headers, SEC_WS_VERSION); const char *protocol = str_map_find (&self->headers, SEC_WS_PROTOCOL); - if (!upgrade || strcmp (upgrade, "websocket") || !version) + struct str tmp; + str_init (&tmp); + bool key_is_valid = base64_decode (key, &tmp) && tmp.len == 16; + str_free (&tmp); + if (!key_is_valid) + FAIL_HANDSHAKE (HTTP_400_BAD_REQUEST, NULL); + + if (!version) FAIL_HANDSHAKE (HTTP_400_BAD_REQUEST, NULL); if (strcmp (version, "13")) FAIL_HANDSHAKE (HTTP_400_BAD_REQUEST, SEC_WS_VERSION ": 13", NULL); @@ -1947,9 +2236,6 @@ ws_handler_push (struct ws_handler *self, const void *data, size_t len) .on_url = ws_handler_on_url, }; - // NOTE: the HTTP parser unfolds values and removes preceeding whitespace, - // but otherwise doesn't touch the values or the following whitespace; - // we might want to strip at least the trailing whitespace size_t n_parsed = http_parser_execute (&self->hp, &http_settings, data, len); From 987eae56610dc0d513130d4ae981b7245ebea220 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Mon, 23 Mar 2015 16:47:21 +0100 Subject: [PATCH 14/79] Steady progress Renamed some constants, added basic UTF-8 validation. --- demo-json-rpc-server.c | 167 +++++++++++++++++++++++++++++++---------- 1 file changed, 126 insertions(+), 41 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index 06f0fc4..806225c 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -144,6 +144,61 @@ isspace_ascii (int c) // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +/// Return a pointer to the next UTF-8 character, or NULL on error +static const char * +utf8_next (const char *s, size_t len) +{ + // End of string, we go no further + if (!len) + return NULL; + + // In the middle of a character -> error + const uint8_t *p = (const unsigned char *) s; + if ((*p & 0xC0) == 0x80) + return NULL; + + // Find out how long the sequence is + unsigned mask = 0xC0; + unsigned tail_len = 0; + while ((*p & mask) == mask) + { + // Invalid start of sequence + if (mask == 0xFE) + return NULL; + + mask |= mask >> 1; + tail_len++; + } + + // Check the rest of the sequence + if (tail_len < --len) + return NULL; + + while (tail_len--) + if ((*++p & 0xC0) != 0x80) + return NULL; + + return (const char *) p; +} + +/// Very rough UTF-8 validation, just makes sure codepoints can be iterated +static bool +utf8_validate (const char *s, size_t len) +{ + const char *next; + while (len) + { + if (!(next = utf8_next (s, len))) + return false; + + len -= next - s; + s = next; + } + return true; +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + static uint8_t g_base64_table[256] = { 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, @@ -1536,23 +1591,22 @@ ws_encode_response_key (const char *key) enum ws_status { - // These names aren't really standard, just somewhat descriptive. - // The RFC isn't really much cleaner about their meaning. + // Named according to the meaning specified in RFC 6455, section 11.2 - WS_STATUS_NORMAL = 1000, - WS_STATUS_GOING_AWAY = 1001, - WS_STATUS_PROTOCOL = 1002, - WS_STATUS_UNACCEPTABLE = 1003, - WS_STATUS_INCONSISTENT = 1007, - WS_STATUS_POLICY = 1008, - WS_STATUS_TOO_BIG = 1009, - WS_STATUS_EXTENSION = 1010, - WS_STATUS_UNEXPECTED = 1011, + WS_STATUS_NORMAL_CLOSURE = 1000, + WS_STATUS_GOING_AWAY = 1001, + WS_STATUS_PROTOCOL_ERROR = 1002, + WS_STATUS_UNSUPPORTED_DATA = 1003, + WS_STATUS_INVALID_PAYLOAD_DATA = 1007, + WS_STATUS_POLICY_VIOLATION = 1008, + WS_STATUS_MESSAGE_TOO_BIG = 1009, + WS_STATUS_MANDATORY_EXTENSION = 1010, + WS_STATUS_INTERNAL_SERVER_ERROR = 1011, // Reserved for internal usage - WS_STATUS_MISSING = 1005, - WS_STATUS_ABNORMAL = 1006, - WS_STATUS_TLS = 1015 + WS_STATUS_NO_STATUS_RECEIVED = 1005, + WS_STATUS_ABNORMAL_CLOSURE = 1006, + WS_STATUS_TLS_HANDSHAKE = 1015 }; // - - Frame parser - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -1624,7 +1678,7 @@ ws_parser_free (struct ws_parser *self) static void ws_parser_unmask (struct ws_parser *self) { - // Yes, this could be made faster. For example by reading the mask in + // This could be made faster. For example by reading the mask in // native byte ordering and applying it directly here. uint64_t end = self->payload_len & ~(uint64_t) 3; @@ -1747,9 +1801,10 @@ ws_parser_push (struct ws_parser *self, const void *data, size_t len) enum ws_handler_state { - WS_HANDLER_HANDSHAKE, ///< Parsing HTTP + WS_HANDLER_CONNECTING, ///< Parsing HTTP WS_HANDLER_OPEN, ///< Parsing WebSockets frames - WS_HANDLER_CLOSING ///< Closing the connection + WS_HANDLER_CLOSING, ///< Closing the connection + WS_HANDLER_CLOSED ///< Dead }; struct ws_handler @@ -1780,27 +1835,27 @@ struct ws_handler ev_timer ping_timer; ///< Ping timer bool received_pong; ///< Received PONG since the last PING + // TODO: void (*on_handshake) (protocols) that will allow the user + // to choose any sub-protocol, if the client has provided any. + + // TODO: "on_connected" after the handshake has finished? + /// Called upon reception of a single full message bool (*on_message) (void *user_data, enum ws_opcode type, const void *data, size_t len); - // TODO: void (*on_initialized) () that will allow the user to choose - // any sub-protocol, if the client has provided any. - - /// The connection has been closed. - /// @a close_code may, or may not, be one of enum ws_status. - // NOTE: the "close_code" is what we receive from the remote endpoint, - // or one of 1005/1006/1015 - // NOTE: the reason is an empty string if omitted + /// The connection has been closed. @a close_code may, or may not, be one + /// of enum ws_status. The @a reason is never NULL. // TODO; also note that ideally, the handler should (be able to) first // receive a notification about the connection being closed because of // an error (recv()) returns -1, and call on_close() in reaction. + // Actually, calling push() could work pretty fine for this. void (*on_close) (void *user_data, int close_code, const char *reason); /// Write a chunk of data to the stream void (*write_cb) (void *user_data, const void *data, size_t len); - // TODO: close_cb + // TODO: "close_cb"; to be used from a ping timer e.g. void *user_data; ///< User data for callbacks }; @@ -1831,6 +1886,9 @@ ws_handler_fail (struct ws_handler *self, enum ws_status reason) // some flag for the case that we're in the middle of ws_handler_push(), // and/or add a mechanism to stop the caller from polling the socket for // reads). + // TODO: set the state to FAILED (not CLOSED as that means the TCP + // connection is closed) and wait until all is sent? + // TODO: make sure we don't send pings after the close } // TODO: ws_handler_close() that behaves like ws_handler_fail() but doesn't @@ -1842,6 +1900,8 @@ static void ws_handler_send (struct ws_handler *self, enum ws_opcode opcode, const void *data, size_t len) { + // TODO: make sure (just assert?) we're in the OPEN state + struct str header; str_init (&header); str_pack_u8 (&header, 0x80 | (opcode & 0x0F)); @@ -1869,15 +1929,19 @@ ws_handler_on_frame_header (void *user_data, const struct ws_parser *parser) { struct ws_handler *self = user_data; + // Note that we aren't expected to send any close frame before closing the + // connection when the frame is unmasked + if (parser->reserved_1 || parser->reserved_2 || parser->reserved_3 || !parser->is_masked // client -> server payload must be masked || (ws_is_control_frame (parser->opcode) && (!parser->is_fin || parser->payload_len > WS_MAX_CONTROL_PAYLOAD_LEN)) || (!ws_is_control_frame (parser->opcode) && - (self->expecting_continuation && parser->opcode != WS_OPCODE_CONT))) - ws_handler_fail (self, WS_STATUS_PROTOCOL); + (self->expecting_continuation && parser->opcode != WS_OPCODE_CONT)) + || parser->payload_len >= 0x8000000000000000ULL) + ws_handler_fail (self, WS_STATUS_PROTOCOL_ERROR); else if (parser->payload_len > self->max_payload_len) - ws_handler_fail (self, WS_STATUS_TOO_BIG); + ws_handler_fail (self, WS_STATUS_MESSAGE_TOO_BIG); else return true; return false; @@ -1891,6 +1955,9 @@ ws_handler_on_control_frame { case WS_OPCODE_CLOSE: // TODO: confirm the close + // TODO: change the state to CLOSING + // TODO: call "on_close" + // NOTE: the reason is an empty string if omitted break; case WS_OPCODE_PING: ws_handler_send_control (self, WS_OPCODE_PONG, @@ -1901,9 +1968,9 @@ ws_handler_on_control_frame self->received_pong = true; break; default: - // TODO: shouldn't we rather fail on unknown control frames? - // But should we actually return false at any time? Yes? - break; + // Unknown control frame + ws_handler_fail (self, WS_STATUS_PROTOCOL_ERROR); + return false; } return true; } @@ -1918,8 +1985,8 @@ ws_handler_on_frame (void *user_data, const struct ws_parser *parser) // TODO: do this rather in "on_frame_header" if (self->message_data.len + parser->payload_len > self->max_payload_len) { - ws_handler_fail (self, WS_STATUS_TOO_BIG); - return true; + ws_handler_fail (self, WS_STATUS_MESSAGE_TOO_BIG); + return false; } if (!self->expecting_continuation) @@ -1932,6 +1999,13 @@ ws_handler_on_frame (void *user_data, const struct ws_parser *parser) if (!parser->is_fin) return true; + if (self->message_opcode == WS_OPCODE_TEXT + && !utf8_validate (self->parser.input.str, self->parser.input.len)) + { + ws_handler_fail (self, WS_STATUS_INVALID_PAYLOAD_DATA); + return false; + } + bool result = self->on_message (self->user_data, self->message_opcode, self->parser.input.str, self->parser.payload_len); str_reset (&self->message_data); @@ -1976,7 +2050,8 @@ ws_handler_init (struct ws_handler *self) str_init (&self->message_data); self->ping_interval = 60; - // This is still ridiculously high + // This is still ridiculously high. Note that the most significant bit + // must always be zero, i.e. the protocol maximum is 0x7FFF FFFF FFFF FFFF. self->max_payload_len = UINT32_MAX; // Just so we can safely stop it @@ -2124,6 +2199,8 @@ ws_handler_http_response (struct ws_handler *self, const char *status, ...) str_vector_free (&v); } +// TODO: also set the connection to some FAILED state or anything that's neither +// CONNECTING nor OPEN #define FAIL_HANDSHAKE(status, ...) \ BLOCK_START \ ws_handler_http_response (self, (status), __VA_ARGS__); \ @@ -2219,14 +2296,22 @@ ws_handler_finish_handshake (struct ws_handler *self) static bool ws_handler_push (struct ws_handler *self, const void *data, size_t len) { - if (self->state != WS_HANDLER_HANDSHAKE) + if (!len) { - // TODO: handle the case of len == 0: - // OPEN: "on_close" WS_STATUS_ABNORMAL - // CLOSING: just close the connection - return ws_parser_push (&self->parser, data, len); + if (self->state == WS_HANDLER_OPEN) + self->on_close (self->user_data, WS_STATUS_ABNORMAL_CLOSURE, ""); + else + { + // TODO: anything to do besides just closing the connection? + } + + self->state = WS_HANDLER_CLOSED; + return false; } + if (self->state != WS_HANDLER_CONNECTING) + return ws_parser_push (&self->parser, data, len); + // The handshake hasn't been done yet, process HTTP headers static const http_parser_settings http_settings = { @@ -3182,7 +3267,7 @@ client_ws_on_message (void *user_data, if (type != WS_OPCODE_TEXT) { - ws_handler_fail (&self->handler, WS_STATUS_UNACCEPTABLE); + ws_handler_fail (&self->handler, WS_STATUS_UNSUPPORTED_DATA); return false; } From 6e9109df4cc781d451ca49ccb9adba9081750651 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Mon, 23 Mar 2015 20:12:02 +0100 Subject: [PATCH 15/79] Don't allow whitespace in base64 --- demo-json-rpc-server.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index 806225c..322d7d2 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -221,7 +221,7 @@ static uint8_t g_base64_table[256] = }; static inline bool -base64_decode_group (const char **s, struct str *output) +base64_decode_group (const char **s, bool ignore_ws, struct str *output) { uint8_t input[4]; size_t loaded = 0; @@ -229,7 +229,7 @@ base64_decode_group (const char **s, struct str *output) { if (!**s) return loaded == 0; - if (!isspace_ascii (**s)) + if (!ignore_ws || !isspace_ascii (**s)) input[loaded++] = **s; } @@ -270,10 +270,10 @@ base64_decode_group (const char **s, struct str *output) } static bool -base64_decode (const char *s, struct str *output) +base64_decode (const char *s, bool ignore_ws, struct str *output) { while (*s) - if (!base64_decode_group (&s, output)) + if (!base64_decode_group (&s, ignore_ws, output)) return false; return true; } @@ -2255,7 +2255,7 @@ ws_handler_finish_handshake (struct ws_handler *self) struct str tmp; str_init (&tmp); - bool key_is_valid = base64_decode (key, &tmp) && tmp.len == 16; + bool key_is_valid = base64_decode (key, false, &tmp) && tmp.len == 16; str_free (&tmp); if (!key_is_valid) FAIL_HANDSHAKE (HTTP_400_BAD_REQUEST, NULL); From 8aa232d32e0f9d751db2f391e69ed71c205555af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Mon, 23 Mar 2015 20:12:53 +0100 Subject: [PATCH 16/79] Add and fix some preliminary tests --- demo-json-rpc-server.c | 256 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 225 insertions(+), 31 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index 322d7d2..8aa070b 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -130,10 +130,12 @@ tolower_ascii_strxfrm (char *dest, const char *src, size_t n) static int strcasecmp_ascii (const char *a, const char *b) { - while (*a && *b) - if (tolower_ascii (*a) != tolower_ascii (*b)) - break; - return *a - *b; + while (*a && tolower_ascii (*a) == tolower_ascii (*b)) + { + a++; + b++; + } + return *(const unsigned char *) a - *(const unsigned char *) b; } static bool @@ -145,6 +147,7 @@ isspace_ascii (int c) // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - /// Return a pointer to the next UTF-8 character, or NULL on error +// TODO: decode the sequence while we're at it static const char * utf8_next (const char *s, size_t len) { @@ -170,18 +173,21 @@ utf8_next (const char *s, size_t len) tail_len++; } + p++; + // Check the rest of the sequence - if (tail_len < --len) + if (tail_len > --len) return NULL; while (tail_len--) - if ((*++p & 0xC0) != 0x80) + if ((*p++ & 0xC0) != 0x80) return NULL; return (const char *) p; } /// Very rough UTF-8 validation, just makes sure codepoints can be iterated +// TODO: also validate the codepoints static bool utf8_validate (const char *s, size_t len) { @@ -1430,6 +1436,8 @@ struct scgi_parser static void scgi_parser_init (struct scgi_parser *self) { + memset (self, 0, sizeof *self); + str_init (&self->input); str_map_init (&self->headers); self->headers.free = free; @@ -1705,6 +1713,7 @@ ws_parser_unmask (struct ws_parser *self) static bool ws_parser_push (struct ws_parser *self, const void *data, size_t len) { + bool success = false; str_append_data (&self->input, data, len); struct msg_unpacker unpacker; @@ -1717,8 +1726,8 @@ ws_parser_push (struct ws_parser *self, const void *data, size_t len) uint16_t u16; case WS_PARSER_FIXED: - if (self->input.len < 2) - return true; + if (unpacker.len - unpacker.offset < 2) + goto need_data; (void) msg_unpacker_u8 (&unpacker, &u8); self->is_fin = (u8 >> 7) & 1; @@ -1737,59 +1746,59 @@ ws_parser_push (struct ws_parser *self, const void *data, size_t len) self->state = WS_PARSER_PAYLOAD_LEN_16; else self->state = WS_PARSER_MASK; - - str_remove_slice (&self->input, 0, 2); break; case WS_PARSER_PAYLOAD_LEN_16: - if (self->input.len < 2) - return true; - - (void) msg_unpacker_u16 (&unpacker, &u16); + if (!msg_unpacker_u16 (&unpacker, &u16)) + goto need_data; self->payload_len = u16; self->state = WS_PARSER_MASK; - str_remove_slice (&self->input, 0, 2); break; case WS_PARSER_PAYLOAD_LEN_64: - if (self->input.len < 8) - return true; - - (void) msg_unpacker_u64 (&unpacker, &self->payload_len); + if (!msg_unpacker_u64 (&unpacker, &self->payload_len)) + goto need_data; self->state = WS_PARSER_MASK; - str_remove_slice (&self->input, 0, 8); break; case WS_PARSER_MASK: if (!self->is_masked) goto end_of_header; - if (self->input.len < 4) - return true; - - (void) msg_unpacker_u32 (&unpacker, &self->mask); - str_remove_slice (&self->input, 0, 4); + if (!msg_unpacker_u32 (&unpacker, &self->mask)) + goto need_data; end_of_header: self->state = WS_PARSER_PAYLOAD; if (!self->on_frame_header (self->user_data, self)) - return false; + goto fail; break; case WS_PARSER_PAYLOAD: - if (self->input.len < self->payload_len) - return true; + // Move the buffer so that payload data is at the front + str_remove_slice (&self->input, 0, unpacker.offset); + // And continue unpacking frames past the payload + msg_unpacker_init (&unpacker, self->input.str, self->input.len); + unpacker.offset = self->payload_len; + + if (self->input.len < self->payload_len) + goto need_data; if (self->is_masked) ws_parser_unmask (self); if (!self->on_frame (self->user_data, self)) - return false; + goto fail; self->state = WS_PARSER_FIXED; - str_reset (&self->input); break; } + +need_data: + success = true; +fail: + str_remove_slice (&self->input, 0, unpacker.offset); + return success; } // - - Server handler - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -2792,6 +2801,7 @@ struct request_handler g_request_handler_json_rpc = static char * canonicalize_url_path (const char *path) { + // XXX: this strips any slashes at the end struct str_vector v; str_vector_init (&v); split_str_ignore_empty (path, '/', &v); @@ -2810,7 +2820,7 @@ canonicalize_url_path (const char *path) if (strcmp (dir, "..")) str_vector_add (&canonical, dir); - else if (canonical.len) + else if (canonical.len > 1) // ".." never goes above the root str_vector_remove (&canonical, canonical.len - 1); } @@ -3631,6 +3641,186 @@ lock_pid_file (struct server_context *ctx, struct error **e) return true; } +// --- Tests ------------------------------------------------------------------- + +static void +test_utf8 (void) +{ + const char valid [] = "2H₂ + O₂ ⇌ 2H₂O, R = 4.7 kΩ, ⌀ 200 mm"; + const char invalid[] = "\xf0\x90\x28\xbc"; + soft_assert ( utf8_validate (valid, sizeof valid)); + soft_assert (!utf8_validate (invalid, sizeof invalid)); +} + +static void +test_base64 (void) +{ + char data[65]; + for (size_t i = 0; i < N_ELEMENTS (data); i++) + data[i] = i; + + struct str encoded; str_init (&encoded); + struct str decoded; str_init (&decoded); + + base64_encode (data, sizeof data, &encoded); + soft_assert (base64_decode (encoded.str, false, &decoded)); + soft_assert (decoded.len == sizeof data); + soft_assert (!memcmp (decoded.str, data, sizeof data)); + + str_free (&encoded); + str_free (&decoded); +} + +static void +test_http_parser (void) +{ + struct str_map parameters; + str_map_init (¶meters); + parameters.key_xfrm = tolower_ascii_strxfrm; + + char *type = NULL; + char *subtype = NULL; + soft_assert (http_parse_media_type ("TEXT/html; CHARset=\"utf\\-8\"", + &type, &subtype, ¶meters)); + soft_assert (!strcasecmp_ascii (type, "text")); + soft_assert (!strcasecmp_ascii (subtype, "html")); + soft_assert (parameters.len == 1); + soft_assert (!strcmp (str_map_find (¶meters, "charset"), "utf-8")); + str_map_free (¶meters); + + struct http_protocol *protocols; + soft_assert (http_parse_upgrade ("websocket, HTTP/2.0, , ", &protocols)); + + soft_assert (!strcmp (protocols->name, "websocket")); + soft_assert (!protocols->version); + + soft_assert (!strcmp (protocols->next->name, "HTTP")); + soft_assert (!strcmp (protocols->next->version, "2.0")); + + soft_assert (!protocols->next->next); + + LIST_FOR_EACH (struct http_protocol, iter, protocols) + http_protocol_destroy (iter); +} + +static bool +test_scgi_parser_on_headers_read (void *user_data) +{ + struct scgi_parser *parser = user_data; + soft_assert (parser->headers.len == 4); + soft_assert (!strcmp (str_map_find (&parser->headers, + "CONTENT_LENGTH"), "27")); + soft_assert (!strcmp (str_map_find (&parser->headers, + "SCGI"), "1")); + soft_assert (!strcmp (str_map_find (&parser->headers, + "REQUEST_METHOD"), "POST")); + soft_assert (!strcmp (str_map_find (&parser->headers, + "REQUEST_URI"), "/deepthought")); + return true; +} + +static bool +test_scgi_parser_on_content (void *user_data, const void *data, size_t len) +{ + (void) user_data; + soft_assert (!strncmp (data, "What is the answer to life?", len)); + return true; +} + +static void +test_scgi_parser (void) +{ + struct scgi_parser parser; + scgi_parser_init (&parser); + parser.on_headers_read = test_scgi_parser_on_headers_read; + parser.on_content = test_scgi_parser_on_content; + parser.user_data = &parser; + + // This is an example straight from the specification + const char example[] = + "70:" + "CONTENT_LENGTH" "\0" "27" "\0" + "SCGI" "\0" "1" "\0" + "REQUEST_METHOD" "\0" "POST" "\0" + "REQUEST_URI" "\0" "/deepthought" "\0" + "," + "What is the answer to life?"; + + soft_assert (scgi_parser_push (&parser, example, sizeof example, NULL)); + scgi_parser_free (&parser); +} + +static bool +test_websockets_on_frame_header (void *user_data, const struct ws_parser *self) +{ + (void) user_data; + soft_assert (self->is_fin); + soft_assert (self->is_masked); + soft_assert (self->opcode == WS_OPCODE_TEXT); + return true; +} + +static bool +test_websockets_on_frame (void *user_data, const struct ws_parser *self) +{ + (void) user_data; + soft_assert (self->input.len == self->payload_len); + soft_assert (!strncmp (self->input.str, "Hello", self->input.len)); + return true; +} + +static void +test_websockets (void) +{ + char *accept = ws_encode_response_key ("dGhlIHNhbXBsZSBub25jZQ=="); + soft_assert (!strcmp (accept, "s3pPLMBiTxaQ9kYGzzhZRbK+xOo=")); + free (accept); + + struct ws_parser parser; + ws_parser_init (&parser); + parser.on_frame_header = test_websockets_on_frame_header; + parser.on_frame = test_websockets_on_frame; + parser.user_data = &parser; + + const char frame[] = "\x81\x85\x37\xfa\x21\x3d\x7f\x9f\x4d\x51\x58"; + soft_assert (ws_parser_push (&parser, frame, sizeof frame - 1)); + ws_parser_free (&parser); + + // TODO: test the server handler (happy path) +} + +static void +test_misc (void) +{ + soft_assert ( validate_json_rpc_content_type + ("application/JSON; charset=\"utf-8\"")); + soft_assert (!validate_json_rpc_content_type + ("text/html; charset=\"utf-8\"")); + + char *canon = canonicalize_url_path ("///../../../etc/./passwd"); + soft_assert (!strcmp (canon, "/etc/passwd")); + free (canon); +} + +int +test_main (int argc, char *argv[]) +{ + struct test test; + test_init (&test, argc, argv); + + test_add_simple (&test, "/utf-8", NULL, test_utf8); + test_add_simple (&test, "/base64", NULL, test_base64); + test_add_simple (&test, "/http-parser", NULL, test_http_parser); + test_add_simple (&test, "/scgi-parser", NULL, test_scgi_parser); + test_add_simple (&test, "/websockets", NULL, test_websockets); + + test_add_simple (&test, "/misc", NULL, test_misc); + + // TODO: write more tests + + return test_run (&test); +} + // --- Main program ------------------------------------------------------------ static void @@ -3685,6 +3875,7 @@ parse_program_arguments (int argc, char **argv) { static const struct opt opts[] = { + { 't', "test", NULL, 0, "self-test" }, { 'd', "debug", NULL, 0, "run in debug mode" }, { 'h', "help", NULL, 0, "display this help and exit" }, { 'V', "version", NULL, 0, "output version information and exit" }, @@ -3701,6 +3892,9 @@ parse_program_arguments (int argc, char **argv) while ((c = opt_handler_get (&oh)) != -1) switch (c) { + case 't': + test_main (argc, argv); + exit (EXIT_SUCCESS); case 'd': g_debug_mode = true; break; From db6dff421667e4e7ab5ea24440f28fb55baeee7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Sun, 29 Mar 2015 03:14:20 +0200 Subject: [PATCH 17/79] Move a lot of stuff to liberty --- demo-json-rpc-server.c | 1510 +--------------------------------------- liberty | 2 +- 2 files changed, 8 insertions(+), 1504 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index 8aa070b..2196217 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -25,6 +25,10 @@ #define print_debug_data ((void *) LOG_DEBUG) #define LIBERTY_WANT_SSL +#define LIBERTY_WANT_PROTO_HTTP +#define LIBERTY_WANT_PROTO_WS +#define LIBERTY_WANT_PROTO_SCGI +#define LIBERTY_WANT_PROTO_FASTCGI #include "config.h" #include "liberty/liberty.c" @@ -42,640 +46,7 @@ // --- Extensions to liberty --------------------------------------------------- -// These should be incorporated into the library ASAP - -#define UNPACKER_INT_BEGIN \ - if (self->len - self->offset < sizeof *value) \ - return false; \ - uint8_t *x = (uint8_t *) self->data + self->offset; \ - self->offset += sizeof *value; - -static bool -msg_unpacker_u16 (struct msg_unpacker *self, uint16_t *value) -{ - UNPACKER_INT_BEGIN - *value - = (uint16_t) x[0] << 24 | (uint16_t) x[1] << 16; - return true; -} - -static bool -msg_unpacker_u32 (struct msg_unpacker *self, uint32_t *value) -{ - UNPACKER_INT_BEGIN - *value - = (uint32_t) x[0] << 24 | (uint32_t) x[1] << 16 - | (uint32_t) x[2] << 8 | (uint32_t) x[3]; - return true; -} - -#undef UNPACKER_INT_BEGIN - -// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -// "msg_writer" should be rewritten on top of this - -static void -str_pack_u8 (struct str *self, uint8_t x) -{ - str_append_data (self, &x, 1); -} - -static void -str_pack_u16 (struct str *self, uint64_t x) -{ - uint8_t tmp[2] = { x >> 8, x }; - str_append_data (self, tmp, sizeof tmp); -} - -static void -str_pack_u32 (struct str *self, uint32_t x) -{ - uint32_t u = x; - uint8_t tmp[4] = { u >> 24, u >> 16, u >> 8, u }; - str_append_data (self, tmp, sizeof tmp); -} - -static void -str_pack_i32 (struct str *self, int32_t x) -{ - str_pack_u32 (self, (uint32_t) x); -} - -static void -str_pack_u64 (struct str *self, uint64_t x) -{ - uint8_t tmp[8] = - { x >> 56, x >> 48, x >> 40, x >> 32, x >> 24, x >> 16, x >> 8, x }; - str_append_data (self, tmp, sizeof tmp); -} - -// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -static int -tolower_ascii (int c) -{ - return c >= 'A' && c <= 'Z' ? c + ('a' - 'A') : c; -} - -static size_t -tolower_ascii_strxfrm (char *dest, const char *src, size_t n) -{ - size_t len = strlen (src); - while (n-- && (*dest++ = tolower_ascii (*src++))) - ; - return len; -} - -static int -strcasecmp_ascii (const char *a, const char *b) -{ - while (*a && tolower_ascii (*a) == tolower_ascii (*b)) - { - a++; - b++; - } - return *(const unsigned char *) a - *(const unsigned char *) b; -} - -static bool -isspace_ascii (int c) -{ - return c == '\f' || c == '\n' || c == '\r' || c == '\t' || c == '\v'; -} - -// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -/// Return a pointer to the next UTF-8 character, or NULL on error -// TODO: decode the sequence while we're at it -static const char * -utf8_next (const char *s, size_t len) -{ - // End of string, we go no further - if (!len) - return NULL; - - // In the middle of a character -> error - const uint8_t *p = (const unsigned char *) s; - if ((*p & 0xC0) == 0x80) - return NULL; - - // Find out how long the sequence is - unsigned mask = 0xC0; - unsigned tail_len = 0; - while ((*p & mask) == mask) - { - // Invalid start of sequence - if (mask == 0xFE) - return NULL; - - mask |= mask >> 1; - tail_len++; - } - - p++; - - // Check the rest of the sequence - if (tail_len > --len) - return NULL; - - while (tail_len--) - if ((*p++ & 0xC0) != 0x80) - return NULL; - - return (const char *) p; -} - -/// Very rough UTF-8 validation, just makes sure codepoints can be iterated -// TODO: also validate the codepoints -static bool -utf8_validate (const char *s, size_t len) -{ - const char *next; - while (len) - { - if (!(next = utf8_next (s, len))) - return false; - - len -= next - s; - s = next; - } - return true; -} - -// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -static uint8_t g_base64_table[256] = -{ - 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 62, 64, 64, 64, 63, - 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 64, 64, 64, 0, 64, 64, - 64, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, - 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 64, 64, 64, 64, 64, - 64, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, - 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 64, 64, 64, 64, 64, - - 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, -}; - -static inline bool -base64_decode_group (const char **s, bool ignore_ws, struct str *output) -{ - uint8_t input[4]; - size_t loaded = 0; - for (; loaded < 4; (*s)++) - { - if (!**s) - return loaded == 0; - if (!ignore_ws || !isspace_ascii (**s)) - input[loaded++] = **s; - } - - size_t len = 3; - if (input[0] == '=' || input[1] == '=') - return false; - if (input[2] == '=' && input[3] != '=') - return false; - if (input[2] == '=') - len--; - if (input[3] == '=') - len--; - - uint8_t a = g_base64_table[input[0]]; - uint8_t b = g_base64_table[input[1]]; - uint8_t c = g_base64_table[input[2]]; - uint8_t d = g_base64_table[input[3]]; - - if (((a | b) | (c | d)) & 0x40) - return false; - - uint32_t block = a << 18 | b << 12 | c << 6 | d; - switch (len) - { - case 1: - str_append_c (output, block >> 16); - break; - case 2: - str_append_c (output, block >> 16); - str_append_c (output, block >> 8); - break; - case 3: - str_append_c (output, block >> 16); - str_append_c (output, block >> 8); - str_append_c (output, block); - } - return true; -} - -static bool -base64_decode (const char *s, bool ignore_ws, struct str *output) -{ - while (*s) - if (!base64_decode_group (&s, ignore_ws, output)) - return false; - return true; -} - -static void -base64_encode (const void *data, size_t len, struct str *output) -{ - const char *alphabet = - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - - const uint8_t *p = data; - size_t n_groups = len / 3; - size_t tail = len - n_groups * 3; - uint32_t group; - - for (; n_groups--; p += 3) - { - group = p[0] << 16 | p[1] << 8 | p[2]; - str_append_c (output, alphabet[(group >> 18) & 63]); - str_append_c (output, alphabet[(group >> 12) & 63]); - str_append_c (output, alphabet[(group >> 6) & 63]); - str_append_c (output, alphabet[ group & 63]); - } - - switch (tail) - { - case 2: - group = p[0] << 16 | p[1] << 8; - str_append_c (output, alphabet[(group >> 18) & 63]); - str_append_c (output, alphabet[(group >> 12) & 63]); - str_append_c (output, alphabet[(group >> 6) & 63]); - str_append_c (output, '='); - break; - case 1: - group = p[0] << 16; - str_append_c (output, alphabet[(group >> 18) & 63]); - str_append_c (output, alphabet[(group >> 12) & 63]); - str_append_c (output, '='); - str_append_c (output, '='); - default: - break; - } -} - -// --- HTTP parsing ------------------------------------------------------------ - -// Basic tokenizer for HTTP header field values, to be used in various parsers. -// The input should already be unwrapped. - -// Recommended literature: -// http://tools.ietf.org/html/rfc7230#section-3.2.6 -// http://tools.ietf.org/html/rfc7230#appendix-B -// http://tools.ietf.org/html/rfc5234#appendix-B.1 - -#define HTTP_TOKENIZER_CLASS(name, definition) \ - static inline bool \ - http_tokenizer_is_ ## name (int c) \ - { \ - return (definition); \ - } - -HTTP_TOKENIZER_CLASS (vchar, c >= 0x21 && c <= 0x7E) -HTTP_TOKENIZER_CLASS (delimiter, !!strchr ("\"(),/:;<=>?@[\\]{}", c)) -HTTP_TOKENIZER_CLASS (whitespace, c == '\t' || c == ' ') -HTTP_TOKENIZER_CLASS (obs_text, c >= 0x80 && c <= 0xFF) - -HTTP_TOKENIZER_CLASS (tchar, - http_tokenizer_is_vchar (c) && !http_tokenizer_is_delimiter (c)) - -HTTP_TOKENIZER_CLASS (qdtext, - c == '\t' || c == ' ' || c == '!' - || (c >= 0x23 && c <= 0x5B) - || (c >= 0x5D && c <= 0x7E) - || http_tokenizer_is_obs_text (c)) - -HTTP_TOKENIZER_CLASS (quoted_pair, - c == '\t' || c == ' ' - || http_tokenizer_is_vchar (c) - || http_tokenizer_is_obs_text (c)) - -#undef HTTP_TOKENIZER_CLASS - -// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -enum http_tokenizer_token -{ - HTTP_T_EOF, ///< Input error - HTTP_T_ERROR, ///< End of input - - HTTP_T_TOKEN, ///< "token" - HTTP_T_QUOTED_STRING, ///< "quoted-string" - HTTP_T_DELIMITER, ///< "delimiters" - HTTP_T_WHITESPACE ///< RWS/OWS/BWS -}; - -struct http_tokenizer -{ - const unsigned char *input; ///< The input string - size_t input_len; ///< Length of the input - size_t offset; ///< Position in the input - - char delimiter; ///< The delimiter character - struct str string; ///< "token" / "quoted-string" content -}; - -static void -http_tokenizer_init (struct http_tokenizer *self, const char *input, size_t len) -{ - memset (self, 0, sizeof *self); - self->input = (const unsigned char *) input; - self->input_len = len; - - str_init (&self->string); -} - -static void -http_tokenizer_free (struct http_tokenizer *self) -{ - str_free (&self->string); -} - -static enum http_tokenizer_token -http_tokenizer_quoted_string (struct http_tokenizer *self) -{ - bool quoted_pair = false; - while (self->offset < self->input_len) - { - int c = self->input[self->offset++]; - if (quoted_pair) - { - if (!http_tokenizer_is_quoted_pair (c)) - return HTTP_T_ERROR; - - str_append_c (&self->string, c); - quoted_pair = false; - } - else if (c == '\\') - quoted_pair = true; - else if (c == '"') - return HTTP_T_QUOTED_STRING; - else if (http_tokenizer_is_qdtext (c)) - str_append_c (&self->string, c); - else - return HTTP_T_ERROR; - } - - // Premature end of input - return HTTP_T_ERROR; -} - -static enum http_tokenizer_token -http_tokenizer_next (struct http_tokenizer *self, bool skip_ows) -{ - str_reset (&self->string); - if (self->offset >= self->input_len) - return HTTP_T_EOF; - - int c = self->input[self->offset++]; - - if (skip_ows) - while (http_tokenizer_is_whitespace (c)) - { - if (self->offset >= self->input_len) - return HTTP_T_EOF; - c = self->input[self->offset++]; - } - - if (c == '"') - return http_tokenizer_quoted_string (self); - - if (http_tokenizer_is_delimiter (c)) - { - self->delimiter = c; - return HTTP_T_DELIMITER; - } - - // Simple variable-length tokens - enum http_tokenizer_token result; - bool (*eater) (int c) = NULL; - if (http_tokenizer_is_whitespace (c)) - { - eater = http_tokenizer_is_whitespace; - result = HTTP_T_WHITESPACE; - } - else if (http_tokenizer_is_tchar (c)) - { - eater = http_tokenizer_is_tchar; - result = HTTP_T_TOKEN; - } - else - return HTTP_T_ERROR; - - str_append_c (&self->string, c); - while (self->offset < self->input_len) - { - if (!eater (c = self->input[self->offset])) - break; - - str_append_c (&self->string, c); - self->offset++; - } - return result; -} - -// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -static bool -http_parse_media_type_parameter - (struct http_tokenizer *t, struct str_map *parameters) -{ - bool result = false; - char *attribute = NULL; - - if (http_tokenizer_next (t, true) != HTTP_T_TOKEN) - goto end; - attribute = xstrdup (t->string.str); - - if (http_tokenizer_next (t, false) != HTTP_T_DELIMITER - || t->delimiter != '=') - goto end; - - switch (http_tokenizer_next (t, false)) - { - case HTTP_T_TOKEN: - case HTTP_T_QUOTED_STRING: - str_map_set (parameters, attribute, xstrdup (t->string.str)); - result = true; - default: - break; - } - -end: - free (attribute); - return result; -} - -/// Parser for "Content-Type". @a type and @a subtype may be non-NULL -/// even if the function fails. @a parameters should be case-insensitive. -static bool -http_parse_media_type (const char *media_type, - char **type, char **subtype, struct str_map *parameters) -{ - bool result = false; - struct http_tokenizer t; - http_tokenizer_init (&t, media_type, strlen (media_type)); - - if (http_tokenizer_next (&t, true) != HTTP_T_TOKEN) - goto end; - *type = xstrdup (t.string.str); - - if (http_tokenizer_next (&t, false) != HTTP_T_DELIMITER - || t.delimiter != '/') - goto end; - - if (http_tokenizer_next (&t, false) != HTTP_T_TOKEN) - goto end; - *subtype = xstrdup (t.string.str); - - while (true) - switch (http_tokenizer_next (&t, true)) - { - case HTTP_T_DELIMITER: - if (t.delimiter != ';') - goto end; - if (!http_parse_media_type_parameter (&t, parameters)) - goto end; - break; - case HTTP_T_EOF: - result = true; - default: - goto end; - } - -end: - http_tokenizer_free (&t); - return result; -} - -// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -struct http_protocol -{ - LIST_HEADER (struct http_protocol) - - char *name; ///< The protocol to upgrade to - char *version; ///< Version of the protocol, if any -}; - -static void -http_protocol_destroy (struct http_protocol *self) -{ - free (self->name); - free (self->version); - free (self); -} - -static bool -http_parse_upgrade (const char *upgrade, struct http_protocol **out) -{ - // HTTP grammar makes this more complicated than it should be - - bool result = false; - struct http_protocol *list = NULL; - struct http_protocol *tail = NULL; - - struct http_tokenizer t; - http_tokenizer_init (&t, upgrade, strlen (upgrade)); - - enum { - STATE_PROTOCOL_NAME, - STATE_SLASH, - STATE_PROTOCOL_VERSION, - STATE_EXPECT_COMMA - } state = STATE_PROTOCOL_NAME; - struct http_protocol *proto = NULL; - - while (true) - switch (state) - { - case STATE_PROTOCOL_NAME: - switch (http_tokenizer_next (&t, false)) - { - case HTTP_T_DELIMITER: - if (t.delimiter != ',') - goto end; - case HTTP_T_WHITESPACE: - break; - case HTTP_T_TOKEN: - proto = xcalloc (1, sizeof *proto); - proto->name = xstrdup (t.string.str); - LIST_APPEND_WITH_TAIL (list, tail, proto); - state = STATE_SLASH; - break; - case HTTP_T_EOF: - result = true; - default: - goto end; - } - break; - case STATE_SLASH: - switch (http_tokenizer_next (&t, false)) - { - case HTTP_T_DELIMITER: - if (t.delimiter == '/') - state = STATE_PROTOCOL_VERSION; - else if (t.delimiter == ',') - state = STATE_PROTOCOL_NAME; - else - goto end; - break; - case HTTP_T_WHITESPACE: - state = STATE_EXPECT_COMMA; - break; - case HTTP_T_EOF: - result = true; - default: - goto end; - } - break; - case STATE_PROTOCOL_VERSION: - switch (http_tokenizer_next (&t, false)) - { - case HTTP_T_TOKEN: - proto->version = xstrdup (t.string.str); - state = STATE_EXPECT_COMMA; - break; - default: - goto end; - } - break; - case STATE_EXPECT_COMMA: - switch (http_tokenizer_next (&t, false)) - { - case HTTP_T_DELIMITER: - if (t.delimiter != ',') - goto end; - state = STATE_PROTOCOL_NAME; - case HTTP_T_WHITESPACE: - break; - case HTTP_T_EOF: - result = true; - default: - goto end; - } - } - -end: - if (result) - *out = list; - else - LIST_FOR_EACH (struct http_protocol, iter, list) - http_protocol_destroy (iter); - - http_tokenizer_free (&t); - return result; -} +// Currently in sync, nothing to be moved. // --- libev helpers ----------------------------------------------------------- @@ -730,322 +101,6 @@ log_message_syslog (void *user_data, const char *quote, const char *fmt, // --- FastCGI ----------------------------------------------------------------- -// Constants from the FastCGI specification document - -#define FCGI_HEADER_LEN 8 - -#define FCGI_VERSION_1 1 -#define FCGI_NULL_REQUEST_ID 0 -#define FCGI_KEEP_CONN 1 - -enum fcgi_type -{ - FCGI_BEGIN_REQUEST = 1, - FCGI_ABORT_REQUEST = 2, - FCGI_END_REQUEST = 3, - FCGI_PARAMS = 4, - FCGI_STDIN = 5, - FCGI_STDOUT = 6, - FCGI_STDERR = 7, - FCGI_DATA = 8, - FCGI_GET_VALUES = 9, - FCGI_GET_VALUES_RESULT = 10, - FCGI_UNKNOWN_TYPE = 11, - FCGI_MAXTYPE = FCGI_UNKNOWN_TYPE -}; - -enum fcgi_role -{ - FCGI_RESPONDER = 1, - FCGI_AUTHORIZER = 2, - FCGI_FILTER = 3 -}; - -enum fcgi_protocol_status -{ - FCGI_REQUEST_COMPLETE = 0, - FCGI_CANT_MPX_CONN = 1, - FCGI_OVERLOADED = 2, - FCGI_UNKNOWN_ROLE = 3 -}; - -#define FCGI_MAX_CONNS "FCGI_MAX_CONNS" -#define FCGI_MAX_REQS "FCGI_MAX_REQS" -#define FCGI_MPXS_CONNS "FCGI_MPXS_CONNS" - -// - - Message stream parser - - - - - - - - - - - - - - - - - - - - - - - - - - - -struct fcgi_parser; - -typedef void (*fcgi_message_fn) - (const struct fcgi_parser *parser, void *user_data); - -enum fcgi_parser_state -{ - FCGI_READING_HEADER, ///< Reading the fixed header portion - FCGI_READING_CONTENT, ///< Reading the message content - FCGI_READING_PADDING ///< Reading the padding -}; - -struct fcgi_parser -{ - enum fcgi_parser_state state; ///< Parsing state - struct str input; ///< Input buffer - - // The next block of fields is considered public: - - uint8_t version; ///< FastCGI protocol version - uint8_t type; ///< FastCGI record type - uint16_t request_id; ///< FastCGI request ID - struct str content; ///< Message data - - uint16_t content_length; ///< Message content length - uint8_t padding_length; ///< Message padding length - - fcgi_message_fn on_message; ///< Callback on message - void *user_data; ///< User data -}; - -static void -fcgi_parser_init (struct fcgi_parser *self) -{ - memset (self, 0, sizeof *self); - str_init (&self->input); - str_init (&self->content); -} - -static void -fcgi_parser_free (struct fcgi_parser *self) -{ - str_free (&self->input); - str_free (&self->content); -} - -static void -fcgi_parser_unpack_header (struct fcgi_parser *self) -{ - struct msg_unpacker unpacker; - msg_unpacker_init (&unpacker, self->input.str, self->input.len); - - bool success = true; - uint8_t reserved; - success &= msg_unpacker_u8 (&unpacker, &self->version); - success &= msg_unpacker_u8 (&unpacker, &self->type); - success &= msg_unpacker_u16 (&unpacker, &self->request_id); - success &= msg_unpacker_u16 (&unpacker, &self->content_length); - success &= msg_unpacker_u8 (&unpacker, &self->padding_length); - success &= msg_unpacker_u8 (&unpacker, &reserved); - hard_assert (success); - - str_remove_slice (&self->input, 0, unpacker.offset); -} - -static void -fcgi_parser_push (struct fcgi_parser *self, const void *data, size_t len) -{ - // This could be made considerably faster for high-throughput applications - // if we use a circular buffer instead of constantly calling memmove() - str_append_data (&self->input, data, len); - - while (true) - switch (self->state) - { - case FCGI_READING_HEADER: - if (self->input.len < FCGI_HEADER_LEN) - return; - - fcgi_parser_unpack_header (self); - self->state = FCGI_READING_CONTENT; - break; - case FCGI_READING_CONTENT: - if (self->input.len < self->content_length) - return; - - // Move an appropriate part of the input buffer to the content buffer - str_reset (&self->content); - str_append_data (&self->content, self->input.str, self->content_length); - str_remove_slice (&self->input, 0, self->content_length); - self->state = FCGI_READING_PADDING; - break; - case FCGI_READING_PADDING: - if (self->input.len < self->padding_length) - return; - - // Call the callback to further process the message - self->on_message (self, self->user_data); - - // Remove the padding from the input buffer - str_remove_slice (&self->input, 0, self->padding_length); - self->state = FCGI_READING_HEADER; - break; - } -} - -// - - Name-value pair parser - - - - - - - - - - - - - - - - - - - - - - - - - - -enum fcgi_nv_parser_state -{ - FCGI_NV_PARSER_NAME_LEN, ///< The first name length octet - FCGI_NV_PARSER_NAME_LEN_FULL, ///< Remaining name length octets - FCGI_NV_PARSER_VALUE_LEN, ///< The first value length octet - FCGI_NV_PARSER_VALUE_LEN_FULL, ///< Remaining value length octets - FCGI_NV_PARSER_NAME, ///< Reading the name - FCGI_NV_PARSER_VALUE ///< Reading the value -}; - -struct fcgi_nv_parser -{ - struct str_map *output; ///< Where the pairs will be stored - - enum fcgi_nv_parser_state state; ///< Parsing state - struct str input; ///< Input buffer - - uint32_t name_len; ///< Length of the name - uint32_t value_len; ///< Length of the value - - char *name; ///< The current name, 0-terminated - char *value; ///< The current value, 0-terminated -}; - -static void -fcgi_nv_parser_init (struct fcgi_nv_parser *self) -{ - memset (self, 0, sizeof *self); - str_init (&self->input); -} - -static void -fcgi_nv_parser_free (struct fcgi_nv_parser *self) -{ - str_free (&self->input); - free (self->name); - free (self->value); -} - -static void -fcgi_nv_parser_push (struct fcgi_nv_parser *self, const void *data, size_t len) -{ - // This could be optimized significantly; I'm not even trying - str_append_data (&self->input, data, len); - - while (true) - { - struct msg_unpacker unpacker; - msg_unpacker_init (&unpacker, self->input.str, self->input.len); - - switch (self->state) - { - uint8_t len; - uint32_t len_full; - - case FCGI_NV_PARSER_NAME_LEN: - if (!msg_unpacker_u8 (&unpacker, &len)) - return; - - if (len >> 7) - self->state = FCGI_NV_PARSER_NAME_LEN_FULL; - else - { - self->name_len = len; - str_remove_slice (&self->input, 0, unpacker.offset); - self->state = FCGI_NV_PARSER_VALUE_LEN; - } - break; - case FCGI_NV_PARSER_NAME_LEN_FULL: - if (!msg_unpacker_u32 (&unpacker, &len_full)) - return; - - self->name_len = len_full & ~(1U << 31); - str_remove_slice (&self->input, 0, unpacker.offset); - self->state = FCGI_NV_PARSER_VALUE_LEN; - break; - case FCGI_NV_PARSER_VALUE_LEN: - if (!msg_unpacker_u8 (&unpacker, &len)) - return; - - if (len >> 7) - self->state = FCGI_NV_PARSER_VALUE_LEN_FULL; - else - { - self->value_len = len; - str_remove_slice (&self->input, 0, unpacker.offset); - self->state = FCGI_NV_PARSER_NAME; - } - break; - case FCGI_NV_PARSER_VALUE_LEN_FULL: - if (!msg_unpacker_u32 (&unpacker, &len_full)) - return; - - self->value_len = len_full & ~(1U << 31); - str_remove_slice (&self->input, 0, unpacker.offset); - self->state = FCGI_NV_PARSER_NAME; - break; - case FCGI_NV_PARSER_NAME: - if (self->input.len < self->name_len) - return; - - self->name = xmalloc (self->name_len + 1); - self->name[self->name_len] = '\0'; - memcpy (self->name, self->input.str, self->name_len); - str_remove_slice (&self->input, 0, self->name_len); - self->state = FCGI_NV_PARSER_VALUE; - break; - case FCGI_NV_PARSER_VALUE: - if (self->input.len < self->value_len) - return; - - self->value = xmalloc (self->value_len + 1); - self->value[self->value_len] = '\0'; - memcpy (self->value, self->input.str, self->value_len); - str_remove_slice (&self->input, 0, self->value_len); - self->state = FCGI_NV_PARSER_NAME_LEN; - - // The map takes ownership of the value - str_map_set (self->output, self->name, self->value); - free (self->name); - - self->name = NULL; - self->value = NULL; - break; - } - } -} - -// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -static void -fcgi_nv_convert_len (size_t len, struct str *output) -{ - if (len < 0x80) - str_pack_u8 (output, len); - else - { - len |= (uint32_t) 1 << 31; - str_pack_u32 (output, len); - } -} - -static void -fcgi_nv_convert (struct str_map *map, struct str *output) -{ - struct str_map_iter iter; - str_map_iter_init (&iter, map); - while (str_map_iter_next (&iter)) - { - const char *name = iter.link->key; - const char *value = iter.link->data; - size_t name_len = iter.link->key_length; - size_t value_len = strlen (value); - - fcgi_nv_convert_len (name_len, output); - fcgi_nv_convert_len (value_len, output); - str_append_data (output, name, name_len); - str_append_data (output, value, value_len); - } -} - -// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - enum fcgi_request_state { FCGI_REQUEST_PARAMS, ///< Reading headers @@ -1401,408 +456,8 @@ fcgi_muxer_push (struct fcgi_muxer *self, const void *data, size_t len) fcgi_parser_push (&self->parser, data, len); } -// --- SCGI -------------------------------------------------------------------- - -enum scgi_parser_state -{ - SCGI_READING_NETSTRING_LENGTH, ///< The length of the header netstring - SCGI_READING_NAME, ///< Header name - SCGI_READING_VALUE, ///< Header value - SCGI_READING_CONTENT ///< Incoming data -}; - -struct scgi_parser -{ - enum scgi_parser_state state; ///< Parsing state - struct str input; ///< Input buffer - - struct str_map headers; ///< Headers parsed - - size_t headers_len; ///< Length of the netstring contents - struct str name; ///< Header name so far - struct str value; ///< Header value so far - - /// Finished parsing request headers. - /// Return false to abort further processing of input. - bool (*on_headers_read) (void *user_data); - - /// Content available; len == 0 means end of file. - /// Return false to abort further processing of input. - bool (*on_content) (void *user_data, const void *data, size_t len); - - void *user_data; ///< User data passed to callbacks -}; - -static void -scgi_parser_init (struct scgi_parser *self) -{ - memset (self, 0, sizeof *self); - - str_init (&self->input); - str_map_init (&self->headers); - self->headers.free = free; - str_init (&self->name); - str_init (&self->value); -} - -static void -scgi_parser_free (struct scgi_parser *self) -{ - str_free (&self->input); - str_map_free (&self->headers); - str_free (&self->name); - str_free (&self->value); -} - -static bool -scgi_parser_push (struct scgi_parser *self, - const void *data, size_t len, struct error **e) -{ - if (!len) - { - if (self->state != SCGI_READING_CONTENT) - { - error_set (e, "premature EOF"); - return false; - } - - // Indicate end of file - return self->on_content (self->user_data, NULL, 0); - } - - // Notice that this madness is significantly harder to parse than FastCGI; - // this procedure could also be optimized significantly - str_append_data (&self->input, data, len); - - bool keep_running = true; - while (keep_running) - switch (self->state) - { - case SCGI_READING_NETSTRING_LENGTH: - { - if (self->input.len < 1) - return true; - - char digit = *self->input.str; - // XXX: this allows for omitting the netstring length altogether - if (digit == ':') - { - self->state = SCGI_READING_NAME; - break; - } - - if (digit < '0' || digit >= '9') - { - error_set (e, "invalid header netstring"); - return false; - } - - size_t new_len = self->headers_len * 10 + (digit - '0'); - if (new_len < self->headers_len) - { - error_set (e, "header netstring is too long"); - return false; - } - self->headers_len = new_len; - str_remove_slice (&self->input, 0, 1); - break; - } - case SCGI_READING_NAME: - { - if (self->input.len < 1) - return true; - - char c = *self->input.str; - if (!self->headers_len) - { - // The netstring is ending but we haven't finished parsing it, - // or the netstring doesn't end with a comma - if (self->name.len || c != ',') - { - error_set (e, "invalid header netstring"); - return false; - } - self->state = SCGI_READING_CONTENT; - keep_running = self->on_headers_read (self->user_data); - } - else if (c != '\0') - str_append_c (&self->name, c); - else - self->state = SCGI_READING_VALUE; - - str_remove_slice (&self->input, 0, 1); - break; - } - case SCGI_READING_VALUE: - { - if (self->input.len < 1) - return true; - - char c = *self->input.str; - if (!self->headers_len) - { - // The netstring is ending but we haven't finished parsing it - error_set (e, "invalid header netstring"); - return false; - } - else if (c != '\0') - str_append_c (&self->value, c); - else - { - // We've got a name-value pair, let's put it in the map - str_map_set (&self->headers, - self->name.str, str_steal (&self->value)); - - str_reset (&self->name); - str_init (&self->value); - - self->state = SCGI_READING_NAME; - } - - str_remove_slice (&self->input, 0, 1); - break; - } - case SCGI_READING_CONTENT: - keep_running = self->on_content - (self->user_data, self->input.str, self->input.len); - str_remove_slice (&self->input, 0, self->input.len); - return keep_running; - } - return false; -} - // --- WebSockets -------------------------------------------------------------- -#define WS_GUID "258EAFA5-E914-47DA-95CA-C5AB0DC85B11" - -#define SEC_WS_KEY "Sec-WebSocket-Key" -#define SEC_WS_ACCEPT "Sec-WebSocket-Accept" -#define SEC_WS_PROTOCOL "Sec-WebSocket-Protocol" -#define SEC_WS_EXTENSIONS "Sec-WebSocket-Extensions" -#define SEC_WS_VERSION "Sec-WebSocket-Version" - -#define WS_MAX_CONTROL_PAYLOAD_LEN 125 - -static char * -ws_encode_response_key (const char *key) -{ - char *response_key = xstrdup_printf ("%s" WS_GUID, key); - unsigned char hash[SHA_DIGEST_LENGTH]; - SHA1 ((unsigned char *) response_key, strlen (response_key), hash); - free (response_key); - - struct str base64; - str_init (&base64); - base64_encode (hash, sizeof hash, &base64); - return str_steal (&base64); -} - -enum ws_status -{ - // Named according to the meaning specified in RFC 6455, section 11.2 - - WS_STATUS_NORMAL_CLOSURE = 1000, - WS_STATUS_GOING_AWAY = 1001, - WS_STATUS_PROTOCOL_ERROR = 1002, - WS_STATUS_UNSUPPORTED_DATA = 1003, - WS_STATUS_INVALID_PAYLOAD_DATA = 1007, - WS_STATUS_POLICY_VIOLATION = 1008, - WS_STATUS_MESSAGE_TOO_BIG = 1009, - WS_STATUS_MANDATORY_EXTENSION = 1010, - WS_STATUS_INTERNAL_SERVER_ERROR = 1011, - - // Reserved for internal usage - WS_STATUS_NO_STATUS_RECEIVED = 1005, - WS_STATUS_ABNORMAL_CLOSURE = 1006, - WS_STATUS_TLS_HANDSHAKE = 1015 -}; - -// - - Frame parser - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -enum ws_parser_state -{ - WS_PARSER_FIXED, ///< Parsing fixed length part - WS_PARSER_PAYLOAD_LEN_16, ///< Parsing extended payload length - WS_PARSER_PAYLOAD_LEN_64, ///< Parsing extended payload length - WS_PARSER_MASK, ///< Parsing masking-key - WS_PARSER_PAYLOAD ///< Parsing payload -}; - -enum ws_opcode -{ - // Non-control - WS_OPCODE_CONT = 0, - WS_OPCODE_TEXT = 1, - WS_OPCODE_BINARY = 2, - - // Control - WS_OPCODE_CLOSE = 8, - WS_OPCODE_PING = 9, - WS_OPCODE_PONG = 10 -}; - -static bool -ws_is_control_frame (int opcode) -{ - return opcode >= WS_OPCODE_CLOSE; -} - -struct ws_parser -{ - struct str input; ///< External input buffer - enum ws_parser_state state; ///< Parsing state - - unsigned is_fin : 1; ///< Final frame of a message? - unsigned is_masked : 1; ///< Is the frame masked? - unsigned reserved_1 : 1; ///< Reserved - unsigned reserved_2 : 1; ///< Reserved - unsigned reserved_3 : 1; ///< Reserved - enum ws_opcode opcode; ///< Opcode - uint32_t mask; ///< Frame mask - uint64_t payload_len; ///< Payload length - - bool (*on_frame_header) (void *user_data, const struct ws_parser *self); - - /// Callback for when a message is successfully parsed. - /// The actual payload is stored in "input", of length "payload_len". - bool (*on_frame) (void *user_data, const struct ws_parser *self); - - void *user_data; ///< User data for callbacks -}; - -static void -ws_parser_init (struct ws_parser *self) -{ - memset (self, 0, sizeof *self); - str_init (&self->input); -} - -static void -ws_parser_free (struct ws_parser *self) -{ - str_free (&self->input); -} - -static void -ws_parser_unmask (struct ws_parser *self) -{ - // This could be made faster. For example by reading the mask in - // native byte ordering and applying it directly here. - - uint64_t end = self->payload_len & ~(uint64_t) 3; - for (uint64_t i = 0; i < end; i += 4) - { - self->input.str[i + 3] ^= self->mask & 0xFF; - self->input.str[i + 2] ^= (self->mask >> 8) & 0xFF; - self->input.str[i + 1] ^= (self->mask >> 16) & 0xFF; - self->input.str[i ] ^= (self->mask >> 24) & 0xFF; - } - - switch (self->payload_len - end) - { - case 3: - self->input.str[end + 2] ^= (self->mask >> 8) & 0xFF; - case 2: - self->input.str[end + 1] ^= (self->mask >> 16) & 0xFF; - case 1: - self->input.str[end ] ^= (self->mask >> 24) & 0xFF; - break; - } -} - -static bool -ws_parser_push (struct ws_parser *self, const void *data, size_t len) -{ - bool success = false; - str_append_data (&self->input, data, len); - - struct msg_unpacker unpacker; - msg_unpacker_init (&unpacker, self->input.str, self->input.len); - - while (true) - switch (self->state) - { - uint8_t u8; - uint16_t u16; - - case WS_PARSER_FIXED: - if (unpacker.len - unpacker.offset < 2) - goto need_data; - - (void) msg_unpacker_u8 (&unpacker, &u8); - self->is_fin = (u8 >> 7) & 1; - self->reserved_1 = (u8 >> 6) & 1; - self->reserved_2 = (u8 >> 5) & 1; - self->reserved_3 = (u8 >> 4) & 1; - self->opcode = u8 & 15; - - (void) msg_unpacker_u8 (&unpacker, &u8); - self->is_masked = (u8 >> 7) & 1; - self->payload_len = u8 & 127; - - if (self->payload_len == 127) - self->state = WS_PARSER_PAYLOAD_LEN_64; - else if (self->payload_len == 126) - self->state = WS_PARSER_PAYLOAD_LEN_16; - else - self->state = WS_PARSER_MASK; - break; - - case WS_PARSER_PAYLOAD_LEN_16: - if (!msg_unpacker_u16 (&unpacker, &u16)) - goto need_data; - self->payload_len = u16; - - self->state = WS_PARSER_MASK; - break; - - case WS_PARSER_PAYLOAD_LEN_64: - if (!msg_unpacker_u64 (&unpacker, &self->payload_len)) - goto need_data; - - self->state = WS_PARSER_MASK; - break; - - case WS_PARSER_MASK: - if (!self->is_masked) - goto end_of_header; - if (!msg_unpacker_u32 (&unpacker, &self->mask)) - goto need_data; - - end_of_header: - self->state = WS_PARSER_PAYLOAD; - if (!self->on_frame_header (self->user_data, self)) - goto fail; - break; - - case WS_PARSER_PAYLOAD: - // Move the buffer so that payload data is at the front - str_remove_slice (&self->input, 0, unpacker.offset); - - // And continue unpacking frames past the payload - msg_unpacker_init (&unpacker, self->input.str, self->input.len); - unpacker.offset = self->payload_len; - - if (self->input.len < self->payload_len) - goto need_data; - if (self->is_masked) - ws_parser_unmask (self); - if (!self->on_frame (self->user_data, self)) - goto fail; - - self->state = WS_PARSER_FIXED; - break; - } - -need_data: - success = true; -fail: - str_remove_slice (&self->input, 0, unpacker.offset); - return success; -} - -// - - Server handler - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - // WebSockets aren't CGI-compatible, therefore we must handle the initial HTTP // handshake ourselves. Luckily it's not too much of a bother with http-parser. // Typically there will be a normal HTTP server in front of us, proxying the @@ -2016,7 +671,7 @@ ws_handler_on_frame (void *user_data, const struct ws_parser *parser) } bool result = self->on_message (self->user_data, self->message_opcode, - self->parser.input.str, self->parser.payload_len); + self->message_data.str, self->message_data.len); str_reset (&self->message_data); return result; } @@ -3643,152 +2298,6 @@ lock_pid_file (struct server_context *ctx, struct error **e) // --- Tests ------------------------------------------------------------------- -static void -test_utf8 (void) -{ - const char valid [] = "2H₂ + O₂ ⇌ 2H₂O, R = 4.7 kΩ, ⌀ 200 mm"; - const char invalid[] = "\xf0\x90\x28\xbc"; - soft_assert ( utf8_validate (valid, sizeof valid)); - soft_assert (!utf8_validate (invalid, sizeof invalid)); -} - -static void -test_base64 (void) -{ - char data[65]; - for (size_t i = 0; i < N_ELEMENTS (data); i++) - data[i] = i; - - struct str encoded; str_init (&encoded); - struct str decoded; str_init (&decoded); - - base64_encode (data, sizeof data, &encoded); - soft_assert (base64_decode (encoded.str, false, &decoded)); - soft_assert (decoded.len == sizeof data); - soft_assert (!memcmp (decoded.str, data, sizeof data)); - - str_free (&encoded); - str_free (&decoded); -} - -static void -test_http_parser (void) -{ - struct str_map parameters; - str_map_init (¶meters); - parameters.key_xfrm = tolower_ascii_strxfrm; - - char *type = NULL; - char *subtype = NULL; - soft_assert (http_parse_media_type ("TEXT/html; CHARset=\"utf\\-8\"", - &type, &subtype, ¶meters)); - soft_assert (!strcasecmp_ascii (type, "text")); - soft_assert (!strcasecmp_ascii (subtype, "html")); - soft_assert (parameters.len == 1); - soft_assert (!strcmp (str_map_find (¶meters, "charset"), "utf-8")); - str_map_free (¶meters); - - struct http_protocol *protocols; - soft_assert (http_parse_upgrade ("websocket, HTTP/2.0, , ", &protocols)); - - soft_assert (!strcmp (protocols->name, "websocket")); - soft_assert (!protocols->version); - - soft_assert (!strcmp (protocols->next->name, "HTTP")); - soft_assert (!strcmp (protocols->next->version, "2.0")); - - soft_assert (!protocols->next->next); - - LIST_FOR_EACH (struct http_protocol, iter, protocols) - http_protocol_destroy (iter); -} - -static bool -test_scgi_parser_on_headers_read (void *user_data) -{ - struct scgi_parser *parser = user_data; - soft_assert (parser->headers.len == 4); - soft_assert (!strcmp (str_map_find (&parser->headers, - "CONTENT_LENGTH"), "27")); - soft_assert (!strcmp (str_map_find (&parser->headers, - "SCGI"), "1")); - soft_assert (!strcmp (str_map_find (&parser->headers, - "REQUEST_METHOD"), "POST")); - soft_assert (!strcmp (str_map_find (&parser->headers, - "REQUEST_URI"), "/deepthought")); - return true; -} - -static bool -test_scgi_parser_on_content (void *user_data, const void *data, size_t len) -{ - (void) user_data; - soft_assert (!strncmp (data, "What is the answer to life?", len)); - return true; -} - -static void -test_scgi_parser (void) -{ - struct scgi_parser parser; - scgi_parser_init (&parser); - parser.on_headers_read = test_scgi_parser_on_headers_read; - parser.on_content = test_scgi_parser_on_content; - parser.user_data = &parser; - - // This is an example straight from the specification - const char example[] = - "70:" - "CONTENT_LENGTH" "\0" "27" "\0" - "SCGI" "\0" "1" "\0" - "REQUEST_METHOD" "\0" "POST" "\0" - "REQUEST_URI" "\0" "/deepthought" "\0" - "," - "What is the answer to life?"; - - soft_assert (scgi_parser_push (&parser, example, sizeof example, NULL)); - scgi_parser_free (&parser); -} - -static bool -test_websockets_on_frame_header (void *user_data, const struct ws_parser *self) -{ - (void) user_data; - soft_assert (self->is_fin); - soft_assert (self->is_masked); - soft_assert (self->opcode == WS_OPCODE_TEXT); - return true; -} - -static bool -test_websockets_on_frame (void *user_data, const struct ws_parser *self) -{ - (void) user_data; - soft_assert (self->input.len == self->payload_len); - soft_assert (!strncmp (self->input.str, "Hello", self->input.len)); - return true; -} - -static void -test_websockets (void) -{ - char *accept = ws_encode_response_key ("dGhlIHNhbXBsZSBub25jZQ=="); - soft_assert (!strcmp (accept, "s3pPLMBiTxaQ9kYGzzhZRbK+xOo=")); - free (accept); - - struct ws_parser parser; - ws_parser_init (&parser); - parser.on_frame_header = test_websockets_on_frame_header; - parser.on_frame = test_websockets_on_frame; - parser.user_data = &parser; - - const char frame[] = "\x81\x85\x37\xfa\x21\x3d\x7f\x9f\x4d\x51\x58"; - soft_assert (ws_parser_push (&parser, frame, sizeof frame - 1)); - ws_parser_free (&parser); - - // TODO: test the server handler (happy path) -} - static void test_misc (void) { @@ -3808,15 +2317,10 @@ test_main (int argc, char *argv[]) struct test test; test_init (&test, argc, argv); - test_add_simple (&test, "/utf-8", NULL, test_utf8); - test_add_simple (&test, "/base64", NULL, test_base64); - test_add_simple (&test, "/http-parser", NULL, test_http_parser); - test_add_simple (&test, "/scgi-parser", NULL, test_scgi_parser); - test_add_simple (&test, "/websockets", NULL, test_websockets); - test_add_simple (&test, "/misc", NULL, test_misc); // TODO: write more tests + // TODO: test the server handler (happy path) return test_run (&test); } diff --git a/liberty b/liberty index 0876458..8c6d187 160000 --- a/liberty +++ b/liberty @@ -1 +1 @@ -Subproject commit 087645848baec5e59e4296817850bd5dd240cbb2 +Subproject commit 8c6d18757d2d4135963f3dbab6d2d5ec8c8b6af3 From 4dbdc849d9b08570ca0ccb4645da3fdc977d6681 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Fri, 10 Apr 2015 01:40:32 +0200 Subject: [PATCH 18/79] Steady progress On the WebSocket service. It's not too far from being finished now. I just have to make some sense of the code again and make sure it's correct. Now that json-rpc-shell should be able to run against this, I can also finally test if both of them work as they should. --- demo-json-rpc-server.c | 238 +++++++++++++++++++++++++++++++---------- 1 file changed, 182 insertions(+), 56 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index 2196217..aa799dd 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -468,6 +468,7 @@ enum ws_handler_state WS_HANDLER_CONNECTING, ///< Parsing HTTP WS_HANDLER_OPEN, ///< Parsing WebSockets frames WS_HANDLER_CLOSING, ///< Closing the connection + WS_HANDLER_ALMOST_DEAD, ///< Closing connection after failure WS_HANDLER_CLOSED ///< Dead }; @@ -475,12 +476,17 @@ struct ws_handler { enum ws_handler_state state; ///< State + // HTTP handshake: + http_parser hp; ///< HTTP parser bool parsing_header_value; ///< Parsing header value or field? struct str field; ///< Field part buffer struct str value; ///< Value part buffer struct str_map headers; ///< HTTP Headers struct str url; ///< Request URL + ev_timer handshake_timeout_watcher; ///< Handshake timeout watcher + + // WebSocket frame protocol: struct ws_parser parser; ///< Protocol frame parser bool expecting_continuation; ///< For non-control traffic @@ -488,27 +494,32 @@ struct ws_handler enum ws_opcode message_opcode; ///< Opcode for the current message struct str message_data; ///< Concatenated message data - unsigned ping_interval; ///< Ping interval in seconds - uint64_t max_payload_len; ///< Maximum length of any message - - // TODO: handshake_timeout - // TODO: a close timer - - // TODO: a ping timer (when no pong is received by the second time the - // timer triggers, it is a ping timeout) ev_timer ping_timer; ///< Ping timer bool received_pong; ///< Received PONG since the last PING + ev_timer close_timeout_watcher; ///< Close timeout watcher + + // Configuration: + + unsigned handshake_timeout; ///< How long to wait for the handshake + unsigned close_timeout; ///< How long to wait for TCP close + unsigned ping_interval; ///< Ping interval in seconds + uint64_t max_payload_len; ///< Maximum length of any message + + // Event callbacks: + // TODO: void (*on_handshake) (protocols) that will allow the user // to choose any sub-protocol, if the client has provided any. + // This may render "on_connected" unnecessary. - // TODO: "on_connected" after the handshake has finished? + /// Called after successfuly connecting (handshake complete) + bool (*on_connected) (void *user_data); /// Called upon reception of a single full message bool (*on_message) (void *user_data, enum ws_opcode type, const void *data, size_t len); - /// The connection has been closed. @a close_code may, or may not, be one + /// The connection is about to close. @a close_code may, or may not, be one /// of enum ws_status. The @a reason is never NULL. // TODO; also note that ideally, the handler should (be able to) first // receive a notification about the connection being closed because of @@ -516,10 +527,13 @@ struct ws_handler // Actually, calling push() could work pretty fine for this. void (*on_close) (void *user_data, int close_code, const char *reason); + // Method callbacks: + /// Write a chunk of data to the stream void (*write_cb) (void *user_data, const void *data, size_t len); - // TODO: "close_cb"; to be used from a ping timer e.g. + /// Close the connection + void (*close_cb) (void *user_data); void *user_data; ///< User data for callbacks }; @@ -541,30 +555,44 @@ ws_handler_send_control (struct ws_handler *self, } static void -ws_handler_fail (struct ws_handler *self, enum ws_status reason) +ws_handler_close (struct ws_handler *self, + enum ws_status close_code, const char *reason, size_t len) { - uint8_t payload[2] = { reason << 8, reason }; - ws_handler_send_control (self, WS_OPCODE_CLOSE, payload, sizeof payload); + struct str payload; + str_init (&payload); + str_pack_u16 (&payload, close_code); + // XXX: maybe accept a null-terminated string on input? Has to be UTF-8 a/w + str_append_data (&payload, reason, len); + ws_handler_send_control (self, WS_OPCODE_CLOSE, payload.str, payload.len); + + // Close initiated by us; the reason is null-terminated within `payload' + if (self->on_close) + self->on_close (self->user_data, close_code, payload.str + 2); + + self->state = WS_HANDLER_CLOSING; + str_free (&payload); +} + +static void +ws_handler_fail (struct ws_handler *self, enum ws_status close_code) +{ + ws_handler_close (self, close_code, NULL, 0); + self->state = WS_HANDLER_ALMOST_DEAD; // TODO: set the close timer, ignore all further incoming input (either set // some flag for the case that we're in the middle of ws_handler_push(), // and/or add a mechanism to stop the caller from polling the socket for // reads). - // TODO: set the state to FAILED (not CLOSED as that means the TCP - // connection is closed) and wait until all is sent? // TODO: make sure we don't send pings after the close } -// TODO: ws_handler_close() that behaves like ws_handler_fail() but doesn't -// ignore frames up to a corresponding close from the client. -// Read the RFC once again to see if we can really process the frames. - // TODO: add support for fragmented responses static void ws_handler_send (struct ws_handler *self, enum ws_opcode opcode, const void *data, size_t len) { - // TODO: make sure (just assert?) we're in the OPEN state + if (!soft_assert (self->state == WS_HANDLER_OPEN)) + return; struct str header; str_init (&header); @@ -611,6 +639,38 @@ ws_handler_on_frame_header (void *user_data, const struct ws_parser *parser) return false; } +static bool +ws_handler_on_protocol_close + (struct ws_handler *self, const struct ws_parser *parser) +{ + struct msg_unpacker unpacker; + msg_unpacker_init (&unpacker, parser->input.str, parser->payload_len); + + char *reason = NULL; + uint16_t close_code = WS_STATUS_NO_STATUS_RECEIVED; + + if (parser->payload_len >= 2) + { + (void) msg_unpacker_u16 (&unpacker, &close_code); + reason = xstrndup (parser->input.str + 2, parser->payload_len - 2); + } + else + reason = xstrdup (""); + + if (self->state != WS_HANDLER_CLOSING) + { + // Close initiated by the client + ws_handler_send_control (self, WS_OPCODE_CLOSE, + parser->input.str, parser->payload_len); + if (self->on_close) + self->on_close (self->user_data, close_code, reason); + } + + free (reason); + self->state = WS_HANDLER_ALMOST_DEAD; + return true; +} + static bool ws_handler_on_control_frame (struct ws_handler *self, const struct ws_parser *parser) @@ -618,11 +678,7 @@ ws_handler_on_control_frame switch (parser->opcode) { case WS_OPCODE_CLOSE: - // TODO: confirm the close - // TODO: change the state to CLOSING - // TODO: call "on_close" - // NOTE: the reason is an empty string if omitted - break; + return ws_handler_on_protocol_close (self, parser); case WS_OPCODE_PING: ws_handler_send_control (self, WS_OPCODE_PONG, parser->input.str, parser->payload_len); @@ -634,6 +690,8 @@ ws_handler_on_control_frame default: // Unknown control frame ws_handler_fail (self, WS_STATUS_PROTOCOL_ERROR); + // FIXME: we shouldn't close the connection right away; + // also check other places return false; } return true; @@ -670,8 +728,10 @@ ws_handler_on_frame (void *user_data, const struct ws_parser *parser) return false; } - bool result = self->on_message (self->user_data, self->message_opcode, - self->message_data.str, self->message_data.len); + bool result = true; + if (self->on_message) + result = self->on_message (self->user_data, self->message_opcode, + self->message_data.str, self->message_data.len); str_reset (&self->message_data); return result; } @@ -686,10 +746,26 @@ ws_handler_on_ping_timer (EV_P_ ev_timer *watcher, int revents) if (!self->received_pong) { // TODO: close/fail the connection? - return; } + else + { + ws_handler_send_control (self, WS_OPCODE_PING, NULL, 0); + ev_timer_again (EV_A_ watcher); + } +} - ws_handler_send_control (self, WS_OPCODE_PING, NULL, 0); +static void +ws_handler_on_close_timeout (EV_P_ ev_timer *watcher, int revents) +{ + struct ws_handler *self = watcher->data; + // TODO: call "close_cb" +} + +static void +ws_handler_on_handshake_timeout (EV_P_ ev_timer *watcher, int revents) +{ + struct ws_handler *self = watcher->data; + // TODO } static void @@ -697,44 +773,61 @@ ws_handler_init (struct ws_handler *self) { memset (self, 0, sizeof *self); + self->state = WS_HANDLER_CONNECTING; + http_parser_init (&self->hp, HTTP_REQUEST); self->hp.data = self; - str_init (&self->field); str_init (&self->value); str_map_init (&self->headers); self->headers.free = free; self->headers.key_xfrm = tolower_ascii_strxfrm; str_init (&self->url); + ev_timer_init (&self->handshake_timeout_watcher, + ws_handler_on_handshake_timeout, 0., 0.); + self->handshake_timeout_watcher.data = self; ws_parser_init (&self->parser); self->parser.on_frame_header = ws_handler_on_frame_header; self->parser.on_frame = ws_handler_on_frame; - str_init (&self->message_data); - self->ping_interval = 60; - // This is still ridiculously high. Note that the most significant bit - // must always be zero, i.e. the protocol maximum is 0x7FFF FFFF FFFF FFFF. - self->max_payload_len = UINT32_MAX; - - // Just so we can safely stop it - ev_timer_init (&self->ping_timer, ws_handler_on_ping_timer, 0., 0.); + ev_timer_init (&self->ping_timer, + ws_handler_on_ping_timer, 0., 0.); + self->ping_timer.data = self; + ev_timer_init (&self->close_timeout_watcher, + ws_handler_on_close_timeout, 0., 0.); self->ping_timer.data = self; // So that the first ping timer doesn't timeout the connection self->received_pong = true; + + self->handshake_timeout = self->close_timeout = self->ping_interval = 60; + // This is still ridiculously high. Note that the most significant bit + // must always be zero, i.e. the protocol maximum is 0x7FFF FFFF FFFF FFFF. + self->max_payload_len = UINT32_MAX; +} + +/// Stop all timers, not going to use the handler anymore +static void +ws_handler_stop (struct ws_handler *self) +{ + ev_timer_stop (EV_DEFAULT_ &self->handshake_timeout_watcher); + ev_timer_stop (EV_DEFAULT_ &self->ping_timer); + ev_timer_stop (EV_DEFAULT_ &self->close_timeout_watcher); } static void ws_handler_free (struct ws_handler *self) { + ws_handler_stop (self); + str_free (&self->field); str_free (&self->value); str_map_free (&self->headers); str_free (&self->url); + ws_parser_free (&self->parser); str_free (&self->message_data); - ev_timer_stop (EV_DEFAULT_ &self->ping_timer); } static bool @@ -863,10 +956,9 @@ ws_handler_http_response (struct ws_handler *self, const char *status, ...) str_vector_free (&v); } -// TODO: also set the connection to some FAILED state or anything that's neither -// CONNECTING nor OPEN #define FAIL_HANDSHAKE(status, ...) \ BLOCK_START \ + self->state = WS_HANDLER_ALMOST_DEAD; \ ws_handler_http_response (self, (status), __VA_ARGS__); \ return false; \ BLOCK_END @@ -949,21 +1041,38 @@ ws_handler_finish_handshake (struct ws_handler *self) str_vector_free (&fields); - // XXX: maybe we should start it earlier so that the handshake can - // timeout as well. ws_handler_connected()? - // - // But it should rather be named "connect_timer" + ev_timer_init (&self->ping_timer, ws_handler_on_ping_timer, + self->ping_interval, 0); ev_timer_start (EV_DEFAULT_ &self->ping_timer); return true; } +/// Tells the handler that the TCP connection has been established so it can +/// timeout when the client handshake doesn't arrive soon enough +static void +ws_handler_start (struct ws_handler *self) +{ + ev_timer_set (&self->handshake_timeout_watcher, + self->handshake_timeout, 0.); + ev_timer_start (EV_DEFAULT_ &self->handshake_timeout_watcher); +} + +/// Push data to the WebSocket handler; "len == 0" means EOF static bool ws_handler_push (struct ws_handler *self, const void *data, size_t len) { + // TODO: make sure all timers are stopped appropriately + if (!len) { + ev_timer_stop (EV_DEFAULT_ &self->handshake_timeout_watcher); + if (self->state == WS_HANDLER_OPEN) - self->on_close (self->user_data, WS_STATUS_ABNORMAL_CLOSURE, ""); + { + if (self->on_close) + self->on_close (self->user_data, + WS_STATUS_ABNORMAL_CLOSURE, ""); + } else { // TODO: anything to do besides just closing the connection? @@ -973,6 +1082,9 @@ ws_handler_push (struct ws_handler *self, const void *data, size_t len) return false; } + if (self->state == WS_HANDLER_ALMOST_DEAD) + // We're waiting for an EOF from the client, must not process data + return true; if (self->state != WS_HANDLER_CONNECTING) return ws_parser_push (&self->parser, data, len); @@ -990,6 +1102,8 @@ ws_handler_push (struct ws_handler *self, const void *data, size_t len) if (self->hp.upgrade) { + ev_timer_stop (EV_DEFAULT_ &self->handshake_timeout_watcher); + // The handshake hasn't been finished, yet there is more data // to be processed after the headers already if (len - n_parsed) @@ -999,12 +1113,16 @@ ws_handler_push (struct ws_handler *self, const void *data, size_t len) return false; self->state = WS_HANDLER_OPEN; + if (self->on_connected) + return self->on_connected (self->user_data); return true; } enum http_errno err = HTTP_PARSER_ERRNO (&self->hp); if (n_parsed != len || err != HPE_OK) { + ev_timer_stop (EV_DEFAULT_ &self->handshake_timeout_watcher); + if (err == HPE_CB_headers_complete) print_debug ("WS handshake failed: %s", "missing `Upgrade' field"); else @@ -1916,13 +2034,6 @@ struct client_ws struct ws_handler handler; ///< WebSockets connection handler }; -static void -client_ws_write (void *user_data, const void *data, size_t len) -{ - struct client *client = user_data; - client_write (client, data, len); -} - static bool client_ws_on_message (void *user_data, enum ws_opcode type, const void *data, size_t len) @@ -1945,6 +2056,20 @@ client_ws_on_message (void *user_data, return true; } +static void +client_ws_write (void *user_data, const void *data, size_t len) +{ + struct client *client = user_data; + client_write (client, data, len); +} + +static void +client_ws_close (void *user_data) +{ + struct client *client = user_data; + client_remove (client); +} + static void client_ws_init (struct client *client) { @@ -1952,10 +2077,10 @@ client_ws_init (struct client *client) client->impl_data = self; ws_handler_init (&self->handler); - self->handler.write_cb = client_ws_write; self->handler.on_message = client_ws_on_message; + self->handler.write_cb = client_ws_write; + self->handler.close_cb = client_ws_close; self->handler.user_data = client; - // TODO: configure the handler some more, e.g. regarding the protocol // One mebibyte seems to be a reasonable value self->handler.max_payload_len = 1 << 10; @@ -2052,6 +2177,7 @@ on_client_ready (EV_P_ ev_io *watcher, int revents) // finished flushing the write queue? This should probably even be // the default behaviour, as it's fairly uncommon for clients to // shutdown the socket for writes while leaving it open for reading. + // Actually, we should wait until the client closes the connection. // TODO: some sort of "on_buffers_flushed" callback for streaming huge // chunks of external (or generated) data. if (!flush_queue (&client->write_queue, watcher)) From 1944f9f17d9e54649ba0f58a85ebbdc20c80b13c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Fri, 10 Apr 2015 01:46:05 +0200 Subject: [PATCH 19/79] Travis CI: Change IRC notification address --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d8e73d0..02f202c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: c notifications: irc: - channels: "anathema.us.nu#anathema" + channels: "anathema.irc.so#anathema" use_notice: true skip_join: true compiler: From 6785d3a9edad87fecdb1219eb9cd5f4c077efc23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Fri, 10 Apr 2015 02:44:13 +0200 Subject: [PATCH 20/79] Implement shutdown --- demo-json-rpc-server.c | 91 +++++++++++++++++++++++++++++++++--------- 1 file changed, 72 insertions(+), 19 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index aa799dd..a65693d 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -1152,6 +1152,7 @@ struct server_context { ev_signal sigterm_watcher; ///< Got SIGTERM ev_signal sigint_watcher; ///< Got SIGINT + ev_timer quit_timeout_watcher; ///< Quit timeout watcher bool quitting; ///< User requested quitting struct listener *listeners; ///< Listeners @@ -1164,6 +1165,11 @@ struct server_context struct str_map config; ///< Server configuration }; +static void initiate_quit (struct server_context *self); +static void try_finish_quit (struct server_context *self); +static void on_quit_timeout (EV_P_ ev_watcher *watcher, int revents); +static void close_listeners (struct server_context *self); + static void server_context_init (struct server_context *self) { @@ -1171,10 +1177,10 @@ server_context_init (struct server_context *self) str_map_init (&self->config); load_config_defaults (&self->config, g_config_table); + ev_timer_init (&self->quit_timeout_watcher, on_quit_timeout, 0., 0.); + self->quit_timeout_watcher.data = self; } -static void close_listeners (struct server_context *self); - static void server_context_free (struct server_context *self) { @@ -1761,14 +1767,8 @@ struct client_impl /// Initialize the client as needed void (*init) (struct client *client); - // TODO: a method for graceful shutdown which will, in the case of - // WebSockets, actually send a "shutdown" close packet, and in the case - // of FastCGI will FCGI_END_REQUEST everything with FCGI_REQUEST_COMPLETE - // and FCGI_OVERLOADED all incoming requests in the meantime (the FastCGI - // specification isn't very clear about how we should respond to this). - // - // We then should set up a timer for about a second until we kill all - // clients for good. + /// Attempt a graceful shutdown + void (*shutdown) (struct client *client); /// Do any additional cleanup void (*destroy) (struct client *client); @@ -1817,6 +1817,8 @@ client_remove (struct client *client) xclose (client->socket_fd); client_free (client); free (client); + + try_finish_quit (ctx); } // - - FastCGI - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -1907,6 +1909,16 @@ client_fcgi_init (struct client *client) self->muxer.user_data = client; } +static void +client_fcgi_shutdown (struct client *client) +{ + struct client_fcgi *self = client->impl_data; + + // TODO: respond with FCGI_END_REQUEST: FCGI_REQUEST_COMPLETE to everything, + // and start sending out FCGI_OVERLOADED to all incoming requests. The + // FastCGI specification isn't very clear about what we should do. +} + static void client_fcgi_destroy (struct client *client) { @@ -1927,9 +1939,10 @@ client_fcgi_push (struct client *client, const void *data, size_t len) static struct client_impl g_client_fcgi = { - .init = client_fcgi_init, - .destroy = client_fcgi_destroy, - .push = client_fcgi_push, + .init = client_fcgi_init, + .shutdown = client_fcgi_shutdown, + .destroy = client_fcgi_destroy, + .push = client_fcgi_push, }; // - - SCGI - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -2086,6 +2099,13 @@ client_ws_init (struct client *client) self->handler.max_payload_len = 1 << 10; } +static void +client_ws_shutdown (struct client *client) +{ + struct client_ws *self = client->impl_data; + ws_handler_close (&self->handler, WS_STATUS_GOING_AWAY, NULL, 0); +} + static void client_ws_destroy (struct client *client) { @@ -2105,12 +2125,12 @@ client_ws_push (struct client *client, const void *data, size_t len) static struct client_impl g_client_ws = { - .init = client_ws_init, - .destroy = client_ws_destroy, - .push = client_ws_push, + .init = client_ws_init, + .shutdown = client_ws_shutdown, + .destroy = client_ws_destroy, + .push = client_ws_push, }; - // --- Basic server stuff ------------------------------------------------------ struct listener @@ -2123,7 +2143,6 @@ struct listener static void close_listeners (struct server_context *self) { - // TODO: factor out the closing act, to be used in initiate_quit() for (size_t i = 0; i < self->n_listeners; i++) { struct listener *listener = &self->listeners[i]; @@ -2136,6 +2155,39 @@ close_listeners (struct server_context *self) } } +static void +try_finish_quit (struct server_context *self) +{ + if (!self->quitting || self->clients) + return; + + ev_timer_stop (EV_DEFAULT_ &self->quit_timeout_watcher); + ev_break (EV_DEFAULT_ EVBREAK_ALL); +} + +static void +on_quit_timeout (EV_P_ ev_watcher *watcher, int revents) +{ + struct server_context *self = watcher->data; + (void) loop; + (void) revents; + + LIST_FOR_EACH (struct client, iter, self->clients) + client_remove (iter); +} + +static void +initiate_quit (struct server_context *self) +{ + close_listeners (self); + LIST_FOR_EACH (struct client, iter, self->clients) + if (iter->impl->shutdown) + iter->impl->shutdown (iter->impl_data); + + ev_timer_set (&self->quit_timeout_watcher, 3., 0.); + self->quitting = true; +} + static bool client_read_loop (EV_P_ struct client *client, ev_io *watcher) { @@ -2217,6 +2269,7 @@ make_client (EV_P_ struct client_impl *impl, int sock_fd) static void on_client_available (EV_P_ ev_io *watcher, int revents) { + struct server_context *ctx = ev_userdata (loop); struct listener *listener = watcher->data; (void) revents; @@ -2235,7 +2288,7 @@ on_client_available (EV_P_ ev_io *watcher, int revents) ev_io_stop (EV_A_ watcher); print_fatal ("%s: %s", "accept", strerror (errno)); - // TODO: initiate_quit (ctx); + initiate_quit (ctx); } // --- Application setup ------------------------------------------------------- From df5b7ad71a56ec6a55b8cd17b90ebaab88c9d35e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Thu, 7 May 2015 20:28:24 +0200 Subject: [PATCH 21/79] Update README --- README | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README b/README index 40594e4..e694d1a 100644 --- a/README +++ b/README @@ -51,6 +51,12 @@ Usage ----- TODO. The main application hasn't been written yet. +Contributing and Support +------------------------ +Use this project's GitHub to report any bugs, request features, or submit pull +requests. If you want to discuss this project, or maybe just hang out with +the developer, feel free to join me at irc://anathema.irc.so, channel #anathema. + License ------- `acid' is written by Přemysl Janouch . From 98bbea72d2565d2a8e69e371f5b2a8a6a5294154 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Sun, 27 Sep 2015 02:10:19 +0200 Subject: [PATCH 22/79] Fix README --- README | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/README b/README index e694d1a..0d9119d 100644 --- a/README +++ b/README @@ -29,11 +29,9 @@ Build dependencies: CMake, pkg-config, help2man, libmagic, liberty (included), http-parser (included) Runtime dependencies: libev, Jansson - $ git clone https://github.com/pjanouch/acid.git - $ git submodule init - $ git submodule update - $ mkdir build - $ cd build + $ git clone --recursive https://github.com/pjanouch/acid.git + $ mkdir acid/build + $ cd acid/build $ cmake .. -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Debug $ make From 3c7b57bba936bf2c120152b73554acb9cfbdb637 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Sun, 27 Sep 2015 02:11:38 +0200 Subject: [PATCH 23/79] Convert README to AsciiDoc --- README => README.adoc | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) rename README => README.adoc (85%) diff --git a/README b/README.adoc similarity index 85% rename from README rename to README.adoc index 0d9119d..bc9d724 100644 --- a/README +++ b/README.adoc @@ -1,7 +1,7 @@ acid ==== -`acid' is A Continuous Integration Daemon. Currently under heavy development. +'acid' is A Continuous Integration Daemon. Currently under heavy development. Right now I'm working on a demo JSON-RPC server that will serve as the basis for the final daemon. @@ -9,12 +9,12 @@ The aim of this project is to provide a dumbed-down alternative to Travis CI. I find it way too complex to set up and run in a local setting, while the basic gist of it is actually very simple -- run some stuff on new git commits. -`acid' will provide a JSON-RPC 2.0 service for frontends over FastCGI, SCGI, or +'acid' will provide a JSON-RPC 2.0 service for frontends over FastCGI, SCGI, or WebSockets, as well as a webhook endpoint for notifications about new commits. The daemon is supposed to be "firewalled" by a normal HTTP server and it will not provide TLS support to secure the communications. -`acid' will be able to tell you about build results via e-mail and/or IRC. +'acid' will be able to tell you about build results via e-mail and/or IRC. Builds will only be supported on the same machine as the daemon. Eventually I might be able to add support for fully replicable builds using Docker. @@ -26,7 +26,7 @@ in a somewhat clean manner. Feel free to contribute. Building and Installing ----------------------- Build dependencies: CMake, pkg-config, help2man, libmagic, - liberty (included), http-parser (included) + liberty (included), http-parser (included) + Runtime dependencies: libev, Jansson $ git clone --recursive https://github.com/pjanouch/acid.git @@ -36,14 +36,16 @@ Runtime dependencies: libev, Jansson $ make To install the application, you can do either the usual: + # make install Or you can try telling CMake to make a package for you. For Debian it is: + $ cpack -G DEB # dpkg -i acid-*.deb -Note that for versions of CMake before 2.8.9, you need to prefix cpack with -`fakeroot' or file ownership will end up wrong. +Note that for versions of CMake before 2.8.9, you need to prefix `cpack` with +`fakeroot` or file ownership will end up wrong. Usage ----- @@ -57,7 +59,7 @@ the developer, feel free to join me at irc://anathema.irc.so, channel #anathema. License ------- -`acid' is written by Přemysl Janouch . +'acid' is written by Přemysl Janouch . You may use the software under the terms of the ISC license, the text of which is included within the package, or, at your option, you may relicense the work From 0f62ef26f564ac3af8f6bf47b646fdc0f12119e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Sat, 16 Jan 2016 06:37:56 +0100 Subject: [PATCH 24/79] Fix FindLibMagic.cmake --- cmake/FindLibMagic.cmake | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmake/FindLibMagic.cmake b/cmake/FindLibMagic.cmake index edae3bf..9bed3b3 100644 --- a/cmake/FindLibMagic.cmake +++ b/cmake/FindLibMagic.cmake @@ -2,5 +2,9 @@ find_library (LIBMAGIC_LIBRARIES magic) find_path (LIBMAGIC_INCLUDE_DIRS magic.h) + +include (FindPackageHandleStandardArgs) find_package_handle_standard_args (LibMagic DEFAULT_MSG LIBMAGIC_LIBRARIES LIBMAGIC_INCLUDE_DIRS) + +mark_as_advanced (LIBMAGIC_LIBRARIES LIBMAGIC_INCLUDE_DIRS) From af3cb3aabad6b236dda5f421c9b694864a02883e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Sat, 16 Jan 2016 06:41:31 +0100 Subject: [PATCH 25/79] Bump liberty --- demo-json-rpc-server.c | 12 ++++++------ liberty | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index a65693d..ab86c59 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -1136,7 +1136,7 @@ ws_handler_push (struct ws_handler *self, const void *data, size_t len) // --- Server ------------------------------------------------------------------ -static struct config_item g_config_table[] = +static struct simple_config_item g_config_table[] = { { "bind_host", NULL, "Address of the server" }, { "port_fastcgi", "9000", "Port to bind for FastCGI" }, @@ -1176,7 +1176,7 @@ server_context_init (struct server_context *self) memset (self, 0, sizeof *self); str_map_init (&self->config); - load_config_defaults (&self->config, g_config_table); + simple_config_load_defaults (&self->config, g_config_table); ev_timer_init (&self->quit_timeout_watcher, on_quit_timeout, 0., 0.); self->quit_timeout_watcher.data = self; } @@ -1583,7 +1583,7 @@ canonicalize_url_path (const char *path) // XXX: this strips any slashes at the end struct str_vector v; str_vector_init (&v); - split_str_ignore_empty (path, '/', &v); + cstr_split_ignore_empty (path, '/', &v); struct str_vector canonical; str_vector_init (&canonical); @@ -2383,7 +2383,7 @@ get_ports_from_config (struct server_context *ctx, { const char *ports; if ((ports = str_map_find (&ctx->config, key))) - split_str_ignore_empty (ports, ',', out); + cstr_split_ignore_empty (ports, ',', out); } static bool @@ -2588,7 +2588,7 @@ parse_program_arguments (int argc, char **argv) printf (PROGRAM_NAME " " PROGRAM_VERSION "\n"); exit (EXIT_SUCCESS); case 'w': - call_write_default_config (optarg, g_config_table); + call_simple_config_write_default (optarg, g_config_table); exit (EXIT_SUCCESS); default: print_error ("wrong options"); @@ -2618,7 +2618,7 @@ main (int argc, char *argv[]) server_context_init (&ctx); struct error *e = NULL; - if (!read_config_file (&ctx.config, &e)) + if (!simple_config_update_from_file (&ctx.config, &e)) { print_error ("error loading configuration: %s", e->message); error_free (e); diff --git a/liberty b/liberty index 8c6d187..8a9a282 160000 --- a/liberty +++ b/liberty @@ -1 +1 @@ -Subproject commit 8c6d18757d2d4135963f3dbab6d2d5ec8c8b6af3 +Subproject commit 8a9a28231bba6334c383dca59f7eccd1e5075693 From 5298d802bb728185855f76afd3d19fde0dd0cf53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Sat, 16 Jan 2016 22:16:01 +0100 Subject: [PATCH 26/79] Fix compiler warning --- demo-json-rpc-server.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index ab86c59..2e0b104 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -1167,7 +1167,7 @@ struct server_context static void initiate_quit (struct server_context *self); static void try_finish_quit (struct server_context *self); -static void on_quit_timeout (EV_P_ ev_watcher *watcher, int revents); +static void on_quit_timeout (EV_P_ ev_timer *watcher, int revents); static void close_listeners (struct server_context *self); static void @@ -2166,7 +2166,7 @@ try_finish_quit (struct server_context *self) } static void -on_quit_timeout (EV_P_ ev_watcher *watcher, int revents) +on_quit_timeout (EV_P_ ev_timer *watcher, int revents) { struct server_context *self = watcher->data; (void) loop; From a95867dbeec78ccd3e377f3cc96a72741d4c12e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Sat, 16 Jan 2016 22:16:25 +0100 Subject: [PATCH 27/79] Fix daemonization --- demo-json-rpc-server.c | 73 ++++++++++++++++++++++++++++++++---------- 1 file changed, 56 insertions(+), 17 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index 2e0b104..483f701 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -44,9 +44,7 @@ #include "http-parser/http_parser.h" -// --- Extensions to liberty --------------------------------------------------- - -// Currently in sync, nothing to be moved. +enum { PIPE_READ, PIPE_WRITE }; // --- libev helpers ----------------------------------------------------------- @@ -2430,21 +2428,21 @@ setup_listen_fds (struct server_context *ctx, struct error **e) return true; } -static bool -lock_pid_file (struct server_context *ctx, struct error **e) +static int +lock_pid_file (const char *path, struct error **e) { - const char *path = str_map_find (&ctx->config, "pid_file"); - if (!path) - return true; - + // When using XDG_RUNTIME_DIR, the file needs to either have its + // access time bumped every 6 hours, or have the sticky bit set int fd = open (path, O_RDWR | O_CREAT, - S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH /* 644 */); + S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH /* 644 */ | S_ISVTX /* sticky */); if (fd < 0) { error_set (e, "can't open `%s': %s", path, strerror (errno)); - return false; + return -1; } + set_cloexec (fd); + struct flock lock = { .l_type = F_WRLCK, @@ -2455,7 +2453,8 @@ lock_pid_file (struct server_context *ctx, struct error **e) if (fcntl (fd, F_SETLK, &lock)) { error_set (e, "can't lock `%s': %s", path, strerror (errno)); - return false; + xclose (fd); + return -1; } struct str pid; @@ -2466,13 +2465,27 @@ lock_pid_file (struct server_context *ctx, struct error **e) || write (fd, pid.str, pid.len) != (ssize_t) pid.len) { error_set (e, "can't write to `%s': %s", path, strerror (errno)); - return false; + xclose (fd); + return -1; } str_free (&pid); // Intentionally not closing the file descriptor; it must stay alive // for the entire life of the application - return true; + return fd; +} + +static bool +app_lock_pid_file (struct server_context *ctx, struct error **e) +{ + const char *path = str_map_find (&ctx->config, "pid_file"); + if (!path) + return true; + + char *resolved = resolve_filename (path, resolve_relative_runtime_filename); + bool result = lock_pid_file (resolved, e) != -1; + free (resolved); + return result; } // --- Tests ------------------------------------------------------------------- @@ -2517,18 +2530,34 @@ on_termination_signal (EV_P_ ev_signal *handle, int revents) } static void -daemonize (void) +daemonize (struct server_context *ctx) { print_status ("daemonizing..."); if (chdir ("/")) exit_fatal ("%s: %s", "chdir", strerror (errno)); + // Because of systemd, we need to exit the parent process _after_ writing + // a PID file, otherwise our grandchild would receive a SIGTERM + int sync_pipe[2]; + if (pipe (sync_pipe)) + exit_fatal ("%s: %s", "pipe", strerror (errno)); + pid_t pid; if ((pid = fork ()) < 0) exit_fatal ("%s: %s", "fork", strerror (errno)); else if (pid) + { + // Wait until all write ends of the pipe are closed, which can mean + // either success or failure, we don't need to care + xclose (sync_pipe[PIPE_WRITE]); + + char dummy; + if (read (sync_pipe[PIPE_READ], &dummy, 1) < 0) + exit_fatal ("%s: %s", "read", strerror (errno)); + exit (EXIT_SUCCESS); + } setsid (); signal (SIGHUP, SIG_IGN); @@ -2541,6 +2570,15 @@ daemonize (void) openlog (PROGRAM_NAME, LOG_NDELAY | LOG_NOWAIT | LOG_PID, 0); g_log_message_real = log_message_syslog; + // Write the PID file (if so configured) and get rid of the pipe, so that + // the read() in our grandparent finally returns zero (no write ends) + struct error *e = NULL; + if (!app_lock_pid_file (ctx, &e)) + exit_fatal ("%s", e->message); + + xclose (sync_pipe[PIPE_READ]); + xclose (sync_pipe[PIPE_WRITE]); + // XXX: we may close our own descriptors this way, crippling ourselves; // there is no real guarantee that we will start with all three // descriptors open. In theory we could try to enumerate the descriptors @@ -2643,7 +2681,6 @@ main (int argc, char *argv[]) LIST_PREPEND (ctx.handlers, &g_request_handler_json_rpc); if (!parse_config (&ctx, &e) - || !lock_pid_file (&ctx, &e) || !setup_listen_fds (&ctx, &e)) { print_error ("%s", e->message); @@ -2652,7 +2689,9 @@ main (int argc, char *argv[]) } if (!g_debug_mode) - daemonize (); + daemonize (&ctx); + else if (!app_lock_pid_file (&ctx, &e)) + exit_fatal ("%s", e->message); ev_run (loop, 0); ev_loop_destroy (loop); From f27315144743d063b8bb58b3c851c393c9108e68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Sat, 16 Jan 2016 22:21:29 +0100 Subject: [PATCH 28/79] Overall revision after a year Use something closer to inheritance for clients --- demo-json-rpc-server.c | 481 +++++++++++++++++++++-------------------- 1 file changed, 246 insertions(+), 235 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index 483f701..00fdb4b 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -1,7 +1,7 @@ /* * demo-json-rpc-server.c: JSON-RPC 2.0 demo server * - * Copyright (c) 2015, Přemysl Janouch + * Copyright (c) 2015 - 2016, Přemysl Janouch * All rights reserved. * * Permission to use, copy, modify, and/or distribute this software for any @@ -756,7 +756,9 @@ static void ws_handler_on_close_timeout (EV_P_ ev_timer *watcher, int revents) { struct ws_handler *self = watcher->data; - // TODO: call "close_cb" + // TODO: anything else to do here? Invalidate our state? + if (self->close_cb) + self->close_cb (self->user_data); } static void @@ -1002,7 +1004,7 @@ ws_handler_finish_handshake (struct ws_handler *self) if (!can_upgrade) FAIL_HANDSHAKE (HTTP_400_BAD_REQUEST, NULL); - // Okay, we've finally got past basic HTTP/1.1 stuff + // Okay, we're finally past the basic HTTP/1.1 stuff const char *key = str_map_find (&self->headers, SEC_WS_KEY); const char *version = str_map_find (&self->headers, SEC_WS_VERSION); const char *protocol = str_map_find (&self->headers, SEC_WS_PROTOCOL); @@ -1733,6 +1735,7 @@ static void request_handler_static_destroy (struct request *request) { (void) request; + // Nothing to dispose of this far } struct request_handler g_request_handler_static = @@ -1748,6 +1751,7 @@ struct client { LIST_HEADER (struct client) + // XXX: do we really need this here? struct server_context *ctx; ///< Server context int socket_fd; ///< The TCP socket @@ -1756,32 +1760,22 @@ struct client ev_io read_watcher; ///< The socket can be read from ev_io write_watcher; ///< The socket can be written to - struct client_impl *impl; ///< Client behaviour - void *impl_data; ///< Client behaviour data + struct client_vtable *vtable; ///< Client behaviour }; -struct client_impl +struct client_vtable { - /// Initialize the client as needed - void (*init) (struct client *client); - /// Attempt a graceful shutdown void (*shutdown) (struct client *client); /// Do any additional cleanup + // TODO: rename to "finalize" or "cleanup"? void (*destroy) (struct client *client); /// Process incoming data; "len == 0" means EOF bool (*push) (struct client *client, const void *data, size_t len); }; -static void -client_init (struct client *self) -{ - memset (self, 0, sizeof *self); - write_queue_init (&self->write_queue); -} - static void client_free (struct client *self) { @@ -1789,40 +1783,116 @@ client_free (struct client *self) } static void -client_write (struct client *client, const void *data, size_t len) +client_write (struct client *self, const void *data, size_t len) { write_req_t *req = xcalloc (1, sizeof *req); req->data.iov_base = memcpy (xmalloc (len), data, len); req->data.iov_len = len; - write_queue_add (&client->write_queue, req); - ev_io_start (EV_DEFAULT_ &client->write_watcher); + write_queue_add (&self->write_queue, req); + ev_io_start (EV_DEFAULT_ &self->write_watcher); } static void -client_remove (struct client *client) +client_destroy (struct client *self) { - struct server_context *ctx = client->ctx; + struct server_context *ctx = self->ctx; - LIST_UNLINK (ctx->clients, client); + LIST_UNLINK (ctx->clients, self); ctx->n_clients--; // First uninitialize the higher-level implementation - client->impl->destroy (client); + self->vtable->destroy (self); - ev_io_stop (EV_DEFAULT_ &client->read_watcher); - ev_io_stop (EV_DEFAULT_ &client->write_watcher); - xclose (client->socket_fd); - client_free (client); - free (client); + ev_io_stop (EV_DEFAULT_ &self->read_watcher); + ev_io_stop (EV_DEFAULT_ &self->write_watcher); + xclose (self->socket_fd); + client_free (self); + free (self); try_finish_quit (ctx); } -// - - FastCGI - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +static bool +client_read_loop (EV_P_ struct client *client, ev_io *watcher) +{ + char buf[8192]; + while (true) + { + ssize_t n_read = recv (watcher->fd, buf, sizeof buf, 0); + if (n_read >= 0) + { + if (!client->vtable->push (client, buf, n_read)) + return false; + if (!n_read) + break; + } + else if (errno == EAGAIN) + return true; + else if (errno != EINTR) + return false; + } + + // Don't receive the EOF condition repeatedly + ev_io_stop (EV_A_ watcher); + + // We can probably still write, so let's just return + // XXX: if there's nothing to be written, shouldn't we close the connection? + return true; +} + +static void +on_client_ready (EV_P_ ev_io *watcher, int revents) +{ + struct client *client = watcher->data; + + if (revents & EV_READ) + if (!client_read_loop (EV_A_ client, watcher)) + goto close; + if (revents & EV_WRITE) + // TODO: add "closing link" functionality -> automatic shutdown + // (half-close) once we manage to flush the write buffer, + // which is logically followed by waiting for an EOF from the client + // TODO: some sort of "on_buffers_flushed" callback for streaming huge + // chunks of external (or generated) data. + if (!flush_queue (&client->write_queue, watcher)) + goto close; + return; + +close: + client_destroy (client); +} + +static void +client_init (EV_P_ struct client *self, int sock_fd) +{ + struct server_context *ctx = ev_userdata (loop); + + memset (self, 0, sizeof *self); + write_queue_init (&self->write_queue); + + set_blocking (sock_fd, false); + self->socket_fd = sock_fd; + + ev_io_init (&self->read_watcher, on_client_ready, sock_fd, EV_READ); + ev_io_init (&self->write_watcher, on_client_ready, sock_fd, EV_WRITE); + self->read_watcher.data = self; + self->write_watcher.data = self; + + // We're only interested in reading as the write queue is empty now + ev_io_start (EV_A_ &self->read_watcher); + + LIST_PREPEND (ctx->clients, self); + ctx->n_clients++; +} + +// --- FastCGI client handler -------------------------------------------------- struct client_fcgi { + struct client client; ///< Parent class struct fcgi_muxer muxer; ///< FastCGI de/multiplexer }; @@ -1832,33 +1902,39 @@ struct client_fcgi_request struct request request; ///< Request }; +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + static void -client_fcgi_request_write (void *user_data, const void *data, size_t len) +client_fcgi_request_write_cb (void *user_data, const void *data, size_t len) { struct client_fcgi_request *request = user_data; fcgi_request_write (request->fcgi_request, data, len); } static void -client_fcgi_request_close (void *user_data) +client_fcgi_request_close_cb (void *user_data) { struct client_fcgi_request *request = user_data; - // TODO: fcgi_request_finish()? That will most probably end up with us - // receiving client_fcgi_request_destroy() + // No more data to send, terminate the substream/request + // XXX: this will most probably end up with client_fcgi_request_destroy(), + // we might or might not need to defer this action + fcgi_request_finish (request->fcgi_request); } +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + static void * client_fcgi_request_start (void *user_data, struct fcgi_request *fcgi_request) { - struct client *client = user_data; + struct client_fcgi *self = user_data; // TODO: what if the request is aborted by ; - struct client_fcgi_request *request = xmalloc (sizeof *request); + struct client_fcgi_request *request = xcalloc (1, sizeof *request); request->fcgi_request = fcgi_request; request_init (&request->request); - request->request.ctx = client->ctx; - request->request.write_cb = client_fcgi_request_write; - request->request.close_cb = client_fcgi_request_close; + request->request.ctx = self->client.ctx; + request->request.write_cb = client_fcgi_request_write_cb; + request->request.close_cb = client_fcgi_request_close_cb; request->request.user_data = request; return request; } @@ -1875,42 +1951,33 @@ client_fcgi_request_destroy (void *handler_data) { struct client_fcgi_request *request = handler_data; request_free (&request->request); - free (handler_data); + free (request); +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +static void +client_fcgi_write_cb (void *user_data, const void *data, size_t len) +{ + struct client_fcgi *self = user_data; + client_write (&self->client, data, len); } static void -client_fcgi_write (void *user_data, const void *data, size_t len) +client_fcgi_close_cb (void *user_data) { - struct client *client = user_data; - client_write (client, data, len); + struct client_fcgi *self = user_data; + // FIXME: we should probably call something like client_shutdown(), + // which may have an argument whether we should really use close() + client_destroy (&self->client); } -static void -client_fcgi_close (void *user_data) -{ - struct client *client = user_data; - client_remove (client); -} - -static void -client_fcgi_init (struct client *client) -{ - struct client_fcgi *self = xcalloc (1, sizeof *self); - client->impl_data = self; - - fcgi_muxer_init (&self->muxer); - self->muxer.write_cb = client_fcgi_write; - self->muxer.close_cb = client_fcgi_close; - self->muxer.request_start_cb = client_fcgi_request_start; - self->muxer.request_push_cb = client_fcgi_request_push; - self->muxer.request_destroy_cb = client_fcgi_request_destroy; - self->muxer.user_data = client; -} +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - static void client_fcgi_shutdown (struct client *client) { - struct client_fcgi *self = client->impl_data; + struct client_fcgi *self = (struct client_fcgi *) client; // TODO: respond with FCGI_END_REQUEST: FCGI_REQUEST_COMPLETE to everything, // and start sending out FCGI_OVERLOADED to all incoming requests. The @@ -1920,65 +1987,79 @@ client_fcgi_shutdown (struct client *client) static void client_fcgi_destroy (struct client *client) { - struct client_fcgi *self = client->impl_data; - client->impl_data = NULL; - + struct client_fcgi *self = (struct client_fcgi *) client; fcgi_muxer_free (&self->muxer); - free (self); } static bool client_fcgi_push (struct client *client, const void *data, size_t len) { - struct client_fcgi *self = client->impl_data; + struct client_fcgi *self = (struct client_fcgi *) client; fcgi_muxer_push (&self->muxer, data, len); return true; } -static struct client_impl g_client_fcgi = +static struct client_vtable client_fcgi_vtable = { - .init = client_fcgi_init, .shutdown = client_fcgi_shutdown, .destroy = client_fcgi_destroy, .push = client_fcgi_push, }; -// - - SCGI - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +static struct client * +client_fcgi_create (EV_P_ int sock_fd) +{ + struct client_fcgi *self = xcalloc (1, sizeof *self); + client_init (EV_A_ &self->client, sock_fd); + self->client.vtable = &client_fcgi_vtable; + + fcgi_muxer_init (&self->muxer); + self->muxer.write_cb = client_fcgi_write_cb; + self->muxer.close_cb = client_fcgi_close_cb; + self->muxer.request_start_cb = client_fcgi_request_start; + self->muxer.request_push_cb = client_fcgi_request_push; + self->muxer.request_destroy_cb = client_fcgi_request_destroy; + self->muxer.user_data = self; + return &self->client; +} + +// --- SCGI client handler ----------------------------------------------------- struct client_scgi { + struct client client; ///< Parent class struct scgi_parser parser; ///< SCGI stream parser struct request request; ///< Request (only one per connection) }; static void -client_scgi_write (void *user_data, const void *data, size_t len) +client_scgi_write_cb (void *user_data, const void *data, size_t len) { - struct client *client = user_data; - client_write (client, data, len); + struct client_scgi *self = user_data; + client_write (&self->client, data, len); } static void -client_scgi_close (void *user_data) +client_scgi_close_cb (void *user_data) { // NOTE: this rather really means "close me [the request]" - struct client *client = user_data; - client_remove (client); + struct client_scgi *self = user_data; + // FIXME: we should probably call something like client_shutdown(), + // which may have an argument whether we should really use close() + client_destroy (&self->client); } static bool client_scgi_on_headers_read (void *user_data) { - struct client *client = user_data; - struct client_scgi *self = client->impl_data; + struct client_scgi *self = user_data; return request_start (&self->request, &self->parser.headers); } static bool client_scgi_on_content (void *user_data, const void *data, size_t len) { - struct client *client = user_data; - struct client_scgi *self = client->impl_data; + struct client_scgi *self = user_data; // XXX: do we have to count CONTENT_LENGTH and supply our own EOF? // If we do produce our own EOF, we should probably make sure we don't @@ -1986,39 +2067,20 @@ client_scgi_on_content (void *user_data, const void *data, size_t len) return request_push (&self->request, data, len); } -static void -client_scgi_init (struct client *client) -{ - struct client_scgi *self = xcalloc (1, sizeof *self); - client->impl_data = self; - - request_init (&self->request); - self->request.ctx = client->ctx; - self->request.write_cb = client_scgi_write; - self->request.close_cb = client_scgi_close; - self->request.user_data = client; - - scgi_parser_init (&self->parser); - self->parser.on_headers_read = client_scgi_on_headers_read; - self->parser.on_content = client_scgi_on_content; - self->parser.user_data = client; -} +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - static void client_scgi_destroy (struct client *client) { - struct client_scgi *self = client->impl_data; - client->impl_data = NULL; - + struct client_scgi *self = (struct client_scgi *) client; request_free (&self->request); scgi_parser_free (&self->parser); - free (self); } static bool client_scgi_push (struct client *client, const void *data, size_t len) { - struct client_scgi *self = client->impl_data; + struct client_scgi *self = (struct client_scgi *) client; struct error *e = NULL; if (scgi_parser_push (&self->parser, data, len, &e)) return true; @@ -2031,17 +2093,37 @@ client_scgi_push (struct client *client, const void *data, size_t len) return false; } -static struct client_impl g_client_scgi = +static struct client_vtable client_scgi_vtable = { - .init = client_scgi_init, .destroy = client_scgi_destroy, .push = client_scgi_push, }; -// - - WebSockets - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +static struct client * +client_scgi_create (EV_P_ int sock_fd) +{ + struct client_scgi *self = xcalloc (1, sizeof *self); + client_init (EV_A_ &self->client, sock_fd); + self->client.vtable = &client_scgi_vtable; + + request_init (&self->request); + self->request.ctx = self->client.ctx; + self->request.write_cb = client_scgi_write_cb; + self->request.close_cb = client_scgi_close_cb; + self->request.user_data = self; + + scgi_parser_init (&self->parser); + self->parser.on_headers_read = client_scgi_on_headers_read; + self->parser.on_content = client_scgi_on_content; + self->parser.user_data = self; + return &self->client; +} + +// --- WebSockets client handler ----------------------------------------------- struct client_ws { + struct client client; ///< Parent class struct ws_handler handler; ///< WebSockets connection handler }; @@ -2049,9 +2131,7 @@ static bool client_ws_on_message (void *user_data, enum ws_opcode type, const void *data, size_t len) { - struct client *client = user_data; - struct client_ws *self = client->impl_data; - + struct client_ws *self = user_data; if (type != WS_OPCODE_TEXT) { ws_handler_fail (&self->handler, WS_STATUS_UNSUPPORTED_DATA); @@ -2060,7 +2140,7 @@ client_ws_on_message (void *user_data, struct str response; str_init (&response); - process_json_rpc (client->ctx, data, len, &response); + process_json_rpc (self->client.ctx, data, len, &response); ws_handler_send (&self->handler, WS_OPCODE_TEXT, response.str, response.len); str_free (&response); @@ -2068,74 +2148,78 @@ client_ws_on_message (void *user_data, } static void -client_ws_write (void *user_data, const void *data, size_t len) +client_ws_write_cb (void *user_data, const void *data, size_t len) { struct client *client = user_data; client_write (client, data, len); } static void -client_ws_close (void *user_data) +client_ws_close_cb (void *user_data) { - struct client *client = user_data; - client_remove (client); + struct client_ws *self = user_data; + // FIXME: we should probably call something like client_shutdown(), + // which may have an argument whether we should really use close() + client_destroy (&self->client); } -static void -client_ws_init (struct client *client) -{ - struct client_ws *self = xmalloc (sizeof *self); - client->impl_data = self; - - ws_handler_init (&self->handler); - self->handler.on_message = client_ws_on_message; - self->handler.write_cb = client_ws_write; - self->handler.close_cb = client_ws_close; - self->handler.user_data = client; - - // One mebibyte seems to be a reasonable value - self->handler.max_payload_len = 1 << 10; -} +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - static void client_ws_shutdown (struct client *client) { - struct client_ws *self = client->impl_data; + struct client_ws *self = (struct client_ws *) client; ws_handler_close (&self->handler, WS_STATUS_GOING_AWAY, NULL, 0); } static void client_ws_destroy (struct client *client) { - struct client_ws *self = client->impl_data; - client->impl_data = NULL; - + struct client_ws *self = (struct client_ws *) client; ws_handler_free (&self->handler); - free (self); } static bool client_ws_push (struct client *client, const void *data, size_t len) { - struct client_ws *self = client->impl_data; + struct client_ws *self = (struct client_ws *) client; return ws_handler_push (&self->handler, data, len); } -static struct client_impl g_client_ws = +static struct client_vtable client_ws_vtable = { - .init = client_ws_init, .shutdown = client_ws_shutdown, .destroy = client_ws_destroy, .push = client_ws_push, }; +static struct client * +client_ws_create (EV_P_ int sock_fd) +{ + struct client_ws *self = xcalloc (1, sizeof *self); + client_init (EV_A_ &self->client, sock_fd); + self->client.vtable = &client_ws_vtable; + + ws_handler_init (&self->handler); + self->handler.on_message = client_ws_on_message; + self->handler.write_cb = client_ws_write_cb; + self->handler.close_cb = client_ws_close_cb; + self->handler.user_data = self; + + // One mebibyte seems to be a reasonable value + self->handler.max_payload_len = 1 << 10; + return &self->client; +} + // --- Basic server stuff ------------------------------------------------------ +typedef struct client *(*client_create_fn) (EV_P_ int sock_fd); + struct listener { int fd; ///< Listening socket FD ev_io watcher; ///< New connection available - struct client_impl *impl; ///< Client behaviour + client_create_fn create; ///< Client constructor }; static void @@ -2171,7 +2255,7 @@ on_quit_timeout (EV_P_ ev_timer *watcher, int revents) (void) revents; LIST_FOR_EACH (struct client, iter, self->clients) - client_remove (iter); + client_destroy (iter); } static void @@ -2179,91 +2263,13 @@ initiate_quit (struct server_context *self) { close_listeners (self); LIST_FOR_EACH (struct client, iter, self->clients) - if (iter->impl->shutdown) - iter->impl->shutdown (iter->impl_data); + if (iter->vtable->shutdown) + iter->vtable->shutdown (iter); ev_timer_set (&self->quit_timeout_watcher, 3., 0.); self->quitting = true; } -static bool -client_read_loop (EV_P_ struct client *client, ev_io *watcher) -{ - char buf[8192]; - while (true) - { - ssize_t n_read = recv (watcher->fd, buf, sizeof buf, 0); - if (n_read >= 0) - { - if (!client->impl->push (client, buf, n_read)) - return false; - if (!n_read) - break; - } - else if (errno == EAGAIN) - return true; - else if (errno != EINTR) - return false; - } - - // Don't receive the EOF condition repeatedly - ev_io_stop (EV_A_ watcher); - - // We can probably still write, so let's just return - return true; -} - -static void -on_client_ready (EV_P_ ev_io *watcher, int revents) -{ - struct client *client = watcher->data; - - if (revents & EV_READ) - if (!client_read_loop (EV_A_ client, watcher)) - goto close; - if (revents & EV_WRITE) - // TODO: shouldn't we at least provide an option (to be used by a client - // implementation if it so desires) to close the connection once we've - // finished flushing the write queue? This should probably even be - // the default behaviour, as it's fairly uncommon for clients to - // shutdown the socket for writes while leaving it open for reading. - // Actually, we should wait until the client closes the connection. - // TODO: some sort of "on_buffers_flushed" callback for streaming huge - // chunks of external (or generated) data. - if (!flush_queue (&client->write_queue, watcher)) - goto close; - return; - -close: - client_remove (client); -} - -static void -make_client (EV_P_ struct client_impl *impl, int sock_fd) -{ - struct server_context *ctx = ev_userdata (loop); - set_blocking (sock_fd, false); - - struct client *client = xmalloc (sizeof *client); - client_init (client); - client->socket_fd = sock_fd; - client->impl = impl; - - ev_io_init (&client->read_watcher, on_client_ready, sock_fd, EV_READ); - ev_io_init (&client->write_watcher, on_client_ready, sock_fd, EV_WRITE); - client->read_watcher.data = client; - client->write_watcher.data = client; - - // We're only interested in reading as the write queue is empty now - ev_io_start (EV_A_ &client->read_watcher); - - // Initialize the higher-level implementation - client->impl->init (client); - - LIST_PREPEND (ctx->clients, client); - ctx->n_clients++; -} - static void on_client_available (EV_P_ ev_io *watcher, int revents) { @@ -2275,7 +2281,7 @@ on_client_available (EV_P_ ev_io *watcher, int revents) { int sock_fd = accept (watcher->fd, NULL, NULL); if (sock_fd != -1) - make_client (EV_A_ listener->impl, sock_fd); + listener->create (EV_A_ sock_fd); else if (errno == EAGAIN) return; else if (errno != EINTR && errno != ECONNABORTED) @@ -2345,7 +2351,7 @@ listener_bind (struct addrinfo *gai_iter) static void listener_add (struct server_context *ctx, const char *host, const char *port, - const struct addrinfo *gai_hints, struct client_impl *impl) + const struct addrinfo *gai_hints, client_create_fn create) { struct addrinfo *gai_result, *gai_iter; int err = getaddrinfo (host, port, gai_hints, &gai_result); @@ -2369,7 +2375,7 @@ listener_add (struct server_context *ctx, const char *host, const char *port, ev_io_init (&listener->watcher, on_client_available, fd, EV_READ); ev_io_start (EV_DEFAULT_ &listener->watcher); listener->watcher.data = listener; - listener->impl = impl; + listener->create = create; break; } freeaddrinfo (gai_result); @@ -2407,13 +2413,13 @@ setup_listen_fds (struct server_context *ctx, struct error **e) for (size_t i = 0; i < ports_fcgi.len; i++) listener_add (ctx, bind_host, ports_fcgi.vector[i], - &gai_hints, &g_client_fcgi); + &gai_hints, client_fcgi_create); for (size_t i = 0; i < ports_scgi.len; i++) listener_add (ctx, bind_host, ports_scgi.vector[i], - &gai_hints, &g_client_scgi); + &gai_hints, client_scgi_create); for (size_t i = 0; i < ports_ws.len; i++) listener_add (ctx, bind_host, ports_ws.vector[i], - &gai_hints, &g_client_ws); + &gai_hints, client_ws_create); str_vector_free (&ports_fcgi); str_vector_free (&ports_scgi); @@ -2526,7 +2532,19 @@ on_termination_signal (EV_P_ ev_signal *handle, int revents) (void) handle; (void) revents; - // TODO: initiate_quit (ctx); + initiate_quit (ctx); +} + +static void +setup_signal_handlers (struct server_context *ctx) +{ + ev_signal_init (&ctx->sigterm_watcher, on_termination_signal, SIGTERM); + ev_signal_start (EV_DEFAULT_ &ctx->sigterm_watcher); + + ev_signal_init (&ctx->sigint_watcher, on_termination_signal, SIGINT); + ev_signal_start (EV_DEFAULT_ &ctx->sigint_watcher); + + (void) signal (SIGPIPE, SIG_IGN); } static void @@ -2668,14 +2686,7 @@ main (int argc, char *argv[]) exit_fatal ("libev initialization failed"); ev_set_userdata (loop, &ctx); - - ev_signal_init (&ctx.sigterm_watcher, on_termination_signal, SIGTERM); - ev_signal_start (EV_DEFAULT_ &ctx.sigterm_watcher); - - ev_signal_init (&ctx.sigint_watcher, on_termination_signal, SIGINT); - ev_signal_start (EV_DEFAULT_ &ctx.sigint_watcher); - - (void) signal (SIGPIPE, SIG_IGN); + setup_signal_handlers (&ctx); LIST_PREPEND (ctx.handlers, &g_request_handler_static); LIST_PREPEND (ctx.handlers, &g_request_handler_json_rpc); From 8b66a3f074e614c678ff8d56f239cb6f1a50deab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Sun, 17 Jan 2016 04:42:16 +0100 Subject: [PATCH 29/79] Bump liberty --- demo-json-rpc-server.c | 55 +++--------------------------------------- liberty | 2 +- 2 files changed, 5 insertions(+), 52 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index 00fdb4b..baa40dd 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -49,10 +49,10 @@ enum { PIPE_READ, PIPE_WRITE }; // --- libev helpers ----------------------------------------------------------- static bool -flush_queue (write_queue_t *queue, ev_io *watcher) +flush_queue (struct write_queue *queue, ev_io *watcher) { struct iovec vec[queue->len], *vec_iter = vec; - for (write_req_t *iter = queue->head; iter; iter = iter->next) + LIST_FOR_EACH (struct write_req, iter, queue->head) *vec_iter++ = iter->data; ssize_t written; @@ -1755,7 +1755,7 @@ struct client struct server_context *ctx; ///< Server context int socket_fd; ///< The TCP socket - write_queue_t write_queue; ///< Write queue + struct write_queue write_queue; ///< Write queue ev_io read_watcher; ///< The socket can be read from ev_io write_watcher; ///< The socket can be written to @@ -1785,7 +1785,7 @@ client_free (struct client *self) static void client_write (struct client *self, const void *data, size_t len) { - write_req_t *req = xcalloc (1, sizeof *req); + struct write_req *req = xcalloc (1, sizeof *req); req->data.iov_base = memcpy (xmalloc (len), data, len); req->data.iov_len = len; @@ -2434,53 +2434,6 @@ setup_listen_fds (struct server_context *ctx, struct error **e) return true; } -static int -lock_pid_file (const char *path, struct error **e) -{ - // When using XDG_RUNTIME_DIR, the file needs to either have its - // access time bumped every 6 hours, or have the sticky bit set - int fd = open (path, O_RDWR | O_CREAT, - S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH /* 644 */ | S_ISVTX /* sticky */); - if (fd < 0) - { - error_set (e, "can't open `%s': %s", path, strerror (errno)); - return -1; - } - - set_cloexec (fd); - - struct flock lock = - { - .l_type = F_WRLCK, - .l_start = 0, - .l_whence = SEEK_SET, - .l_len = 0, - }; - if (fcntl (fd, F_SETLK, &lock)) - { - error_set (e, "can't lock `%s': %s", path, strerror (errno)); - xclose (fd); - return -1; - } - - struct str pid; - str_init (&pid); - str_append_printf (&pid, "%ld", (long) getpid ()); - - if (ftruncate (fd, 0) - || write (fd, pid.str, pid.len) != (ssize_t) pid.len) - { - error_set (e, "can't write to `%s': %s", path, strerror (errno)); - xclose (fd); - return -1; - } - str_free (&pid); - - // Intentionally not closing the file descriptor; it must stay alive - // for the entire life of the application - return fd; -} - static bool app_lock_pid_file (struct server_context *ctx, struct error **e) { diff --git a/liberty b/liberty index 8a9a282..f213a76 160000 --- a/liberty +++ b/liberty @@ -1 +1 @@ -Subproject commit 8a9a28231bba6334c383dca59f7eccd1e5075693 +Subproject commit f213a76ad494efe150a786b195a744e4b87c5ca9 From 7b94a03e8c304682871e2e6e229934e3d493d208 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Mon, 14 Mar 2016 21:18:27 +0100 Subject: [PATCH 30/79] Update IRC server address --- .travis.yml | 2 +- README.adoc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 02f202c..e0edcfd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: c notifications: irc: - channels: "anathema.irc.so#anathema" + channels: "irc.janouch.name#dev" use_notice: true skip_join: true compiler: diff --git a/README.adoc b/README.adoc index bc9d724..3781b97 100644 --- a/README.adoc +++ b/README.adoc @@ -55,7 +55,7 @@ Contributing and Support ------------------------ Use this project's GitHub to report any bugs, request features, or submit pull requests. If you want to discuss this project, or maybe just hang out with -the developer, feel free to join me at irc://anathema.irc.so, channel #anathema. +the developer, feel free to join me at irc://irc.janouch.name, channel #dev. License ------- From 369f94f5ab6d94e72bdf286021858d83ac470659 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Fri, 3 Feb 2017 22:38:56 +0100 Subject: [PATCH 31/79] Travis CI: brevify notifications --- .travis.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.travis.yml b/.travis.yml index e0edcfd..7395b90 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,12 @@ notifications: channels: "irc.janouch.name#dev" use_notice: true skip_join: true + template: + - "%{repository_name}#%{build_number} on %{branch}: %{message}" + - " %{compare_url}" + - " %{build_url}" + on_success: change + on_failure: always compiler: - clang - gcc From 733451cf2ab997ec1c1bbcba0d2263c2f1fa13de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Fri, 3 Feb 2017 22:39:28 +0100 Subject: [PATCH 32/79] Fix and update LICENSE --- LICENSE | 23 +++++++++++------------ demo-json-rpc-server.c | 1 - 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/LICENSE b/LICENSE index 553e875..8cc3add 100644 --- a/LICENSE +++ b/LICENSE @@ -1,14 +1,13 @@ - Copyright (c) 2015, Přemysl Janouch - All rights reserved. +Copyright (c) 2015, Přemysl Janouch - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY - SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION - OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN - CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION +OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index baa40dd..10fe925 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -2,7 +2,6 @@ * demo-json-rpc-server.c: JSON-RPC 2.0 demo server * * Copyright (c) 2015 - 2016, Přemysl Janouch - * All rights reserved. * * Permission to use, copy, modify, and/or distribute this software for any * purpose with or without fee is hereby granted, provided that the above From 0ec06857149d17ad0e86bb66f2fe9418bed13d8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Sun, 5 Feb 2017 22:44:01 +0100 Subject: [PATCH 33/79] Bump liberty --- demo-json-rpc-server.c | 56 +++++++++++++++++++++--------------------- liberty | 2 +- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index 10fe925..88f1adc 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -939,20 +939,20 @@ ws_handler_http_responsev (struct ws_handler *self, static void ws_handler_http_response (struct ws_handler *self, const char *status, ...) { - struct str_vector v; - str_vector_init (&v); + struct strv v; + strv_init (&v); va_list ap; va_start (ap, status); const char *s; while ((s = va_arg (ap, const char *))) - str_vector_add (&v, s); + strv_append (&v, s); va_end (ap); ws_handler_http_responsev (self, status, v.vector); - str_vector_free (&v); + strv_free (&v); } #define FAIL_HANDSHAKE(status, ...) \ @@ -1020,16 +1020,16 @@ ws_handler_finish_handshake (struct ws_handler *self) if (strcmp (version, "13")) FAIL_HANDSHAKE (HTTP_400_BAD_REQUEST, SEC_WS_VERSION ": 13", NULL); - struct str_vector fields; - str_vector_init (&fields); + struct strv fields; + strv_init (&fields); - str_vector_add_args (&fields, + strv_append_args (&fields, "Upgrade: websocket", "Connection: Upgrade", NULL); char *response_key = ws_encode_response_key (key); - str_vector_add_owned (&fields, + strv_append_owned (&fields, xstrdup_printf (SEC_WS_ACCEPT ": %s", response_key)); free (response_key); @@ -1038,7 +1038,7 @@ ws_handler_finish_handshake (struct ws_handler *self) ws_handler_http_responsev (self, HTTP_101_SWITCHING_PROTOCOLS, fields.vector); - str_vector_free (&fields); + strv_free (&fields); ev_timer_init (&self->ping_timer, ws_handler_on_ping_timer, self->ping_interval, 0); @@ -1580,15 +1580,15 @@ static char * canonicalize_url_path (const char *path) { // XXX: this strips any slashes at the end - struct str_vector v; - str_vector_init (&v); - cstr_split_ignore_empty (path, '/', &v); + struct strv v; + strv_init (&v); + cstr_split (path, "/", true, &v); - struct str_vector canonical; - str_vector_init (&canonical); + struct strv canonical; + strv_init (&canonical); // So that the joined path always begins with a slash - str_vector_add (&canonical, ""); + strv_append (&canonical, ""); for (size_t i = 0; i < v.len; i++) { @@ -1597,15 +1597,15 @@ canonicalize_url_path (const char *path) continue; if (strcmp (dir, "..")) - str_vector_add (&canonical, dir); + strv_append (&canonical, dir); else if (canonical.len > 1) // ".." never goes above the root - str_vector_remove (&canonical, canonical.len - 1); + strv_remove (&canonical, canonical.len - 1); } - str_vector_free (&v); + strv_free (&v); - char *joined = join_str_vector (&canonical, '/'); - str_vector_free (&canonical); + char *joined = strv_join (&canonical, "/"); + strv_free (&canonical); return joined; } @@ -2382,11 +2382,11 @@ listener_add (struct server_context *ctx, const char *host, const char *port, static void get_ports_from_config (struct server_context *ctx, - const char *key, struct str_vector *out) + const char *key, struct strv *out) { const char *ports; if ((ports = str_map_find (&ctx->config, key))) - cstr_split_ignore_empty (ports, ',', out); + cstr_split (ports, ",", true, out); } static bool @@ -2398,9 +2398,9 @@ setup_listen_fds (struct server_context *ctx, struct error **e) .ai_flags = AI_PASSIVE, }; - struct str_vector ports_fcgi; str_vector_init (&ports_fcgi); - struct str_vector ports_scgi; str_vector_init (&ports_scgi); - struct str_vector ports_ws; str_vector_init (&ports_ws); + struct strv ports_fcgi; strv_init (&ports_fcgi); + struct strv ports_scgi; strv_init (&ports_scgi); + struct strv ports_ws; strv_init (&ports_ws); get_ports_from_config (ctx, "port_fastcgi", &ports_fcgi); get_ports_from_config (ctx, "port_scgi", &ports_scgi); @@ -2420,9 +2420,9 @@ setup_listen_fds (struct server_context *ctx, struct error **e) listener_add (ctx, bind_host, ports_ws.vector[i], &gai_hints, client_ws_create); - str_vector_free (&ports_fcgi); - str_vector_free (&ports_scgi); - str_vector_free (&ports_ws); + strv_free (&ports_fcgi); + strv_free (&ports_scgi); + strv_free (&ports_ws); if (!ctx->n_listeners) { diff --git a/liberty b/liberty index f213a76..084e964 160000 --- a/liberty +++ b/liberty @@ -1 +1 @@ -Subproject commit f213a76ad494efe150a786b195a744e4b87c5ca9 +Subproject commit 084e964286bfcd13ee6a25a2ee35dfba9da1072e From 2b7d4554717a282d1e78a72dab6aa8e924b37b86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Mon, 6 Feb 2017 17:18:24 +0100 Subject: [PATCH 34/79] Fix quitting --- demo-json-rpc-server.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index 88f1adc..a528b60 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -1176,7 +1176,7 @@ server_context_init (struct server_context *self) str_map_init (&self->config); simple_config_load_defaults (&self->config, g_config_table); - ev_timer_init (&self->quit_timeout_watcher, on_quit_timeout, 0., 0.); + ev_timer_init (&self->quit_timeout_watcher, on_quit_timeout, 3., 0.); self->quit_timeout_watcher.data = self; } @@ -2260,13 +2260,15 @@ on_quit_timeout (EV_P_ ev_timer *watcher, int revents) static void initiate_quit (struct server_context *self) { + self->quitting = true; close_listeners (self); + + // Wait a little while for all clients to clean up, if necessary LIST_FOR_EACH (struct client, iter, self->clients) if (iter->vtable->shutdown) iter->vtable->shutdown (iter); - - ev_timer_set (&self->quit_timeout_watcher, 3., 0.); - self->quitting = true; + ev_timer_start (EV_DEFAULT_ &self->quit_timeout_watcher); + try_finish_quit (self); } static void From a785dc96708d8edb5f2ae786a8deee7798523762 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Mon, 6 Feb 2017 18:28:40 +0100 Subject: [PATCH 35/79] WebSocket: fix header parsing --- demo-json-rpc-server.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index a528b60..b2d804d 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -476,7 +476,7 @@ struct ws_handler // HTTP handshake: http_parser hp; ///< HTTP parser - bool parsing_header_value; ///< Parsing header value or field? + bool have_header_value; ///< Parsing header value or field? struct str field; ///< Field part buffer struct str value; ///< Value part buffer struct str_map headers; ///< HTTP Headers @@ -873,14 +873,14 @@ static int ws_handler_on_header_field (http_parser *parser, const char *at, size_t len) { struct ws_handler *self = parser->data; - if (self->parsing_header_value) + if (self->have_header_value) { ws_handler_on_header_read (self); str_reset (&self->field); str_reset (&self->value); } str_append_data (&self->field, at, len); - self->parsing_header_value = false; + self->have_header_value = false; return 0; } @@ -889,13 +889,17 @@ ws_handler_on_header_value (http_parser *parser, const char *at, size_t len) { struct ws_handler *self = parser->data; str_append_data (&self->value, at, len); - self->parsing_header_value = true; + self->have_header_value = true; return 0; } static int ws_handler_on_headers_complete (http_parser *parser) { + struct ws_handler *self = parser->data; + if (self->have_header_value) + ws_handler_on_header_read (self); + // We strictly require a protocol upgrade if (!parser->upgrade) return 2; From c4ebf2ccd57c3c095b58de5f6c33019177306712 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Mon, 6 Feb 2017 18:28:53 +0100 Subject: [PATCH 36/79] Fix segfault on missing Sec-WebSocket-Key --- demo-json-rpc-server.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index b2d804d..11ebfdf 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -1012,6 +1012,9 @@ ws_handler_finish_handshake (struct ws_handler *self) const char *version = str_map_find (&self->headers, SEC_WS_VERSION); const char *protocol = str_map_find (&self->headers, SEC_WS_PROTOCOL); + if (!key) + FAIL_HANDSHAKE (HTTP_400_BAD_REQUEST, NULL); + struct str tmp; str_init (&tmp); bool key_is_valid = base64_decode (key, false, &tmp) && tmp.len == 16; From 2986f6cda0a7834d6ad5d7e927b58dd5ba4a2502 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Mon, 6 Feb 2017 18:30:02 +0100 Subject: [PATCH 37/79] Fix segfault on client destruction etc. --- demo-json-rpc-server.c | 1 + 1 file changed, 1 insertion(+) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index 11ebfdf..47622ba 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -1877,6 +1877,7 @@ client_init (EV_P_ struct client *self, int sock_fd) struct server_context *ctx = ev_userdata (loop); memset (self, 0, sizeof *self); + self->ctx = ctx; write_queue_init (&self->write_queue); set_blocking (sock_fd, false); From e6f9e532291f6b4ec0a89c31f317446ac1674f0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Mon, 6 Feb 2017 18:51:52 +0100 Subject: [PATCH 38/79] Fix segfault in WebSocket parsing --- demo-json-rpc-server.c | 1 + 1 file changed, 1 insertion(+) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index 47622ba..2e855bf 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -789,6 +789,7 @@ ws_handler_init (struct ws_handler *self) ws_parser_init (&self->parser); self->parser.on_frame_header = ws_handler_on_frame_header; self->parser.on_frame = ws_handler_on_frame; + self->parser.user_data = self; str_init (&self->message_data); ev_timer_init (&self->ping_timer, From d35e733c6ef9ac43639c1d1361446ecb3e94b6f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Mon, 6 Feb 2017 19:47:24 +0100 Subject: [PATCH 39/79] Bump liberty Fixing a bug in the WebSocket frame parser. --- liberty | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/liberty b/liberty index 084e964..9afcb33 160000 --- a/liberty +++ b/liberty @@ -1 +1 @@ -Subproject commit 084e964286bfcd13ee6a25a2ee35dfba9da1072e +Subproject commit 9afcb337ada91f87aa1a566ec3feba1a12bc9287 From b85d1d74a435ca706f7285d47bbfd8b48e8778e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Mon, 6 Feb 2017 20:48:14 +0100 Subject: [PATCH 40/79] Don't respond to notifications --- demo-json-rpc-server.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index 2e855bf..85b54a9 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -2149,8 +2149,9 @@ client_ws_on_message (void *user_data, struct str response; str_init (&response); process_json_rpc (self->client.ctx, data, len, &response); - ws_handler_send (&self->handler, - WS_OPCODE_TEXT, response.str, response.len); + if (response.len) + ws_handler_send (&self->handler, + WS_OPCODE_TEXT, response.str, response.len); str_free (&response); return true; } From 419147beecc8bd9e62aa370422ef8808f67d42f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Tue, 20 Jun 2017 14:01:10 +0200 Subject: [PATCH 41/79] Update README --- README.adoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.adoc b/README.adoc index 3781b97..e8f0404 100644 --- a/README.adoc +++ b/README.adoc @@ -57,6 +57,8 @@ Use this project's GitHub to report any bugs, request features, or submit pull requests. If you want to discuss this project, or maybe just hang out with the developer, feel free to join me at irc://irc.janouch.name, channel #dev. +Bitcoin donations: 12r5uEWEgcHC46xd64tt3hHt9EUvYYDHe9 + License ------- 'acid' is written by Přemysl Janouch . From ca90e9df83409ccbc3a7d79295718e3572352c36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Tue, 20 Jun 2017 14:01:23 +0200 Subject: [PATCH 42/79] Fix teardown --- demo-json-rpc-server.c | 1 + 1 file changed, 1 insertion(+) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index 85b54a9..c8ce29d 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -2387,6 +2387,7 @@ listener_add (struct server_context *ctx, const char *host, const char *port, ev_io_start (EV_DEFAULT_ &listener->watcher); listener->watcher.data = listener; listener->create = create; + listener->fd = fd; break; } freeaddrinfo (gai_result); From 5e88608286089cbcce81fd8fa9217cf77a1f7576 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Tue, 20 Jun 2017 14:04:40 +0200 Subject: [PATCH 43/79] Bump liberty and http-parser --- http-parser | 2 +- liberty | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/http-parser b/http-parser index d547f3b..1b79aba 160000 --- a/http-parser +++ b/http-parser @@ -1 +1 @@ -Subproject commit d547f3b1a98ed07fdcdaf401a8cbc5fffe9bfa6c +Subproject commit 1b79abab34d4763c0467f1173a406ad2817c1635 diff --git a/liberty b/liberty index 9afcb33..1dcd259 160000 --- a/liberty +++ b/liberty @@ -1 +1 @@ -Subproject commit 9afcb337ada91f87aa1a566ec3feba1a12bc9287 +Subproject commit 1dcd259d0506b9e2de3715bdf07144b22f57903a From 333efdc70f89f5582cf21976d127e17ecd5c5b03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Sun, 24 Jun 2018 00:15:53 +0200 Subject: [PATCH 44/79] CMakeLists.txt: fix variable name --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 97c63cb..895a6e5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,10 +2,10 @@ project (acid C) cmake_minimum_required (VERSION 2.8.5) # Moar warnings -if ("${CMAKE_C_COMPILER_ID}" MATCHES "GNU" OR CMAKE_COMPILER_IS_GNUC) +if ("${CMAKE_C_COMPILER_ID}" MATCHES "GNU" OR CMAKE_COMPILER_IS_GNUCC) # -Wunused-function is pretty annoying here, as everything is static set (CMAKE_C_FLAGS "-std=c99 -Wall -Wextra -Wno-unused-function") -endif ("${CMAKE_C_COMPILER_ID}" MATCHES "GNU" OR CMAKE_COMPILER_IS_GNUC) +endif ("${CMAKE_C_COMPILER_ID}" MATCHES "GNU" OR CMAKE_COMPILER_IS_GNUCC) # Version set (project_VERSION_MAJOR "0") From 4078c8845ce4549edb23e3138e45d32af8858ee5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Sun, 24 Jun 2018 00:21:10 +0200 Subject: [PATCH 45/79] Relicense to 0BSD, update mail address I've come to the conclusion that copyright mostly just stands in the way of software development. In my jurisdiction I cannot give up my own copyright and 0BSD seems to be the closest thing to public domain. The updated mail address, also used in my author/committer lines, is shorter and looks nicer. People rarely interact anyway. --- CMakeLists.txt | 2 +- LICENSE | 5 ++--- README.adoc | 9 ++------- demo-json-rpc-server.c | 5 ++--- 4 files changed, 7 insertions(+), 14 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 895a6e5..c78736a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -72,7 +72,7 @@ endforeach (page) # CPack set (CPACK_PACKAGE_DESCRIPTION_SUMMARY "A Continuous Integration Daemon") set (CPACK_PACKAGE_VENDOR "Premysl Janouch") -set (CPACK_PACKAGE_CONTACT "Přemysl Janouch ") +set (CPACK_PACKAGE_CONTACT "Přemysl Janouch ") set (CPACK_RESOURCE_FILE_LICENSE "${PROJECT_SOURCE_DIR}/LICENSE") set (CPACK_PACKAGE_VERSION_MAJOR ${project_VERSION_MAJOR}) set (CPACK_PACKAGE_VERSION_MINOR ${project_VERSION_MINOR}) diff --git a/LICENSE b/LICENSE index 8cc3add..01dc408 100644 --- a/LICENSE +++ b/LICENSE @@ -1,8 +1,7 @@ -Copyright (c) 2015, Přemysl Janouch +Copyright (c) 2015 - 2018, Přemysl Janouch Permission to use, copy, modify, and/or distribute this software for any -purpose with or without fee is hereby granted, provided that the above -copyright notice and this permission notice appear in all copies. +purpose with or without fee is hereby granted. THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF diff --git a/README.adoc b/README.adoc index e8f0404..2f80d73 100644 --- a/README.adoc +++ b/README.adoc @@ -61,10 +61,5 @@ Bitcoin donations: 12r5uEWEgcHC46xd64tt3hHt9EUvYYDHe9 License ------- -'acid' is written by Přemysl Janouch . - -You may use the software under the terms of the ISC license, the text of which -is included within the package, or, at your option, you may relicense the work -under the MIT or the Modified BSD License, as listed at the following site: - -http://www.gnu.org/licenses/license-list.html +This software is released under the terms of the 0BSD license, the text of which +is included within the package along with the list of authors. diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index c8ce29d..10c808e 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -1,11 +1,10 @@ /* * demo-json-rpc-server.c: JSON-RPC 2.0 demo server * - * Copyright (c) 2015 - 2016, Přemysl Janouch + * Copyright (c) 2015 - 2018, Přemysl Janouch * * Permission to use, copy, modify, and/or distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. + * purpose with or without fee is hereby granted. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF From b312c022ae28e0852cba7430465971a2a391f016 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Sun, 24 Jun 2018 00:21:32 +0200 Subject: [PATCH 46/79] Update README --- README.adoc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.adoc b/README.adoc index 2f80d73..80d708f 100644 --- a/README.adoc +++ b/README.adoc @@ -29,7 +29,7 @@ Build dependencies: CMake, pkg-config, help2man, libmagic, liberty (included), http-parser (included) + Runtime dependencies: libev, Jansson - $ git clone --recursive https://github.com/pjanouch/acid.git + $ git clone --recursive https://git.janouch.name/p/acid.git $ mkdir acid/build $ cd acid/build $ cmake .. -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Debug @@ -53,11 +53,11 @@ TODO. The main application hasn't been written yet. Contributing and Support ------------------------ -Use this project's GitHub to report any bugs, request features, or submit pull -requests. If you want to discuss this project, or maybe just hang out with -the developer, feel free to join me at irc://irc.janouch.name, channel #dev. +Use https://git.janouch.name/p/acid to report any bugs, request features, +or submit pull requests. `git send-email` is tolerated. If you want to discuss +the project, feel free to join me at ircs://irc.janouch.name, channel #dev. -Bitcoin donations: 12r5uEWEgcHC46xd64tt3hHt9EUvYYDHe9 +Bitcoin donations are accepted at: 12r5uEWEgcHC46xd64tt3hHt9EUvYYDHe9 License ------- From 131debe985199e3d261a985b2055a86fc4bd3fdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Sun, 24 Jun 2018 00:40:10 +0200 Subject: [PATCH 47/79] Bump liberty --- .gitmodules | 2 +- demo-json-rpc-server.c | 106 ++++++++++++++++------------------------- liberty | 2 +- 3 files changed, 44 insertions(+), 66 deletions(-) diff --git a/.gitmodules b/.gitmodules index c8d5acf..1a41faf 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "liberty"] path = liberty - url = git://github.com/pjanouch/liberty.git + url = https://git.janouch.name/p/liberty.git [submodule "http-parser"] path = http-parser url = https://github.com/joyent/http-parser.git diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index 10c808e..0cfb13b 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -144,8 +144,7 @@ fcgi_muxer_send (struct fcgi_muxer *self, { hard_assert (len <= UINT16_MAX); - struct str message; - str_init (&message); + struct str message = str_make (); str_pack_u8 (&message, FCGI_VERSION_1); str_pack_u8 (&message, type); @@ -177,10 +176,9 @@ fcgi_request_init (struct fcgi_request *self) { memset (self, 0, sizeof *self); - str_map_init (&self->headers); - self->headers.free = free; + self->headers = str_map_make (free); - fcgi_nv_parser_init (&self->hdr_parser); + self->hdr_parser = fcgi_nv_parser_make (); self->hdr_parser.output = &self->headers; } @@ -275,17 +273,15 @@ static void fcgi_muxer_on_get_values (struct fcgi_muxer *self, const struct fcgi_parser *parser) { - struct str_map values; str_map_init (&values); values.free = free; - struct str_map response; str_map_init (&response); response.free = free; + struct str_map values = str_map_make (free); + struct str_map response = str_map_make (free); - struct fcgi_nv_parser nv_parser; - fcgi_nv_parser_init (&nv_parser); + struct fcgi_nv_parser nv_parser = fcgi_nv_parser_make (); nv_parser.output = &values; fcgi_nv_parser_push (&nv_parser, parser->content.str, parser->content.len); - struct str_map_iter iter; - str_map_iter_init (&iter, &values); + struct str_map_iter iter = str_map_iter_make (&values); while (str_map_iter_next (&iter)) { const char *key = iter.link->key; @@ -297,8 +293,7 @@ fcgi_muxer_on_get_values str_map_set (&response, key, xstrdup ("1")); } - struct str content; - str_init (&content); + struct str content = str_make (); fcgi_nv_convert (&response, &content); fcgi_muxer_send (self, FCGI_GET_VALUES_RESULT, parser->request_id, content.str, content.len); @@ -312,8 +307,8 @@ static void fcgi_muxer_on_begin_request (struct fcgi_muxer *self, const struct fcgi_parser *parser) { - struct msg_unpacker unpacker; - msg_unpacker_init (&unpacker, parser->content.str, parser->content.len); + struct msg_unpacker unpacker = + msg_unpacker_make (parser->content.str, parser->content.len); uint16_t role; uint8_t flags; @@ -435,7 +430,7 @@ fcgi_muxer_on_message (const struct fcgi_parser *parser, void *user_data) static void fcgi_muxer_init (struct fcgi_muxer *self) { - fcgi_parser_init (&self->parser); + self->parser = fcgi_parser_make (); self->parser.on_message = fcgi_muxer_on_message; self->parser.user_data = self; } @@ -554,8 +549,7 @@ static void ws_handler_close (struct ws_handler *self, enum ws_status close_code, const char *reason, size_t len) { - struct str payload; - str_init (&payload); + struct str payload = str_make (); str_pack_u16 (&payload, close_code); // XXX: maybe accept a null-terminated string on input? Has to be UTF-8 a/w str_append_data (&payload, reason, len); @@ -590,8 +584,7 @@ ws_handler_send (struct ws_handler *self, if (!soft_assert (self->state == WS_HANDLER_OPEN)) return; - struct str header; - str_init (&header); + struct str header = str_make (); str_pack_u8 (&header, 0x80 | (opcode & 0x0F)); if (len > UINT16_MAX) @@ -639,8 +632,8 @@ static bool ws_handler_on_protocol_close (struct ws_handler *self, const struct ws_parser *parser) { - struct msg_unpacker unpacker; - msg_unpacker_init (&unpacker, parser->input.str, parser->payload_len); + struct msg_unpacker unpacker = + msg_unpacker_make (parser->input.str, parser->payload_len); char *reason = NULL; uint16_t close_code = WS_STATUS_NO_STATUS_RECEIVED; @@ -775,21 +768,20 @@ ws_handler_init (struct ws_handler *self) http_parser_init (&self->hp, HTTP_REQUEST); self->hp.data = self; - str_init (&self->field); - str_init (&self->value); - str_map_init (&self->headers); - self->headers.free = free; + self->field = str_make (); + self->value = str_make (); + self->headers = str_map_make (free); self->headers.key_xfrm = tolower_ascii_strxfrm; - str_init (&self->url); + self->url = str_make (); ev_timer_init (&self->handshake_timeout_watcher, ws_handler_on_handshake_timeout, 0., 0.); self->handshake_timeout_watcher.data = self; - ws_parser_init (&self->parser); + self->parser = ws_parser_make (); self->parser.on_frame_header = ws_handler_on_frame_header; self->parser.on_frame = ws_handler_on_frame; self->parser.user_data = self; - str_init (&self->message_data); + self->message_data = str_make (); ev_timer_init (&self->ping_timer, ws_handler_on_ping_timer, 0., 0.); @@ -927,8 +919,7 @@ ws_handler_http_responsev (struct ws_handler *self, { hard_assert (status != NULL); - struct str response; - str_init (&response); + struct str response = str_make (); str_append_printf (&response, "HTTP/1.1 %s\r\n", status); while (*fields) @@ -943,8 +934,7 @@ ws_handler_http_responsev (struct ws_handler *self, static void ws_handler_http_response (struct ws_handler *self, const char *status, ...) { - struct strv v; - strv_init (&v); + struct strv v = strv_make (); va_list ap; va_start (ap, status); @@ -1015,8 +1005,7 @@ ws_handler_finish_handshake (struct ws_handler *self) if (!key) FAIL_HANDSHAKE (HTTP_400_BAD_REQUEST, NULL); - struct str tmp; - str_init (&tmp); + struct str tmp = str_make (); bool key_is_valid = base64_decode (key, false, &tmp) && tmp.len == 16; str_free (&tmp); if (!key_is_valid) @@ -1027,9 +1016,7 @@ ws_handler_finish_handshake (struct ws_handler *self) if (strcmp (version, "13")) FAIL_HANDSHAKE (HTTP_400_BAD_REQUEST, SEC_WS_VERSION ": 13", NULL); - struct strv fields; - strv_init (&fields); - + struct strv fields = strv_make (); strv_append_args (&fields, "Upgrade: websocket", "Connection: Upgrade", @@ -1181,7 +1168,7 @@ server_context_init (struct server_context *self) { memset (self, 0, sizeof *self); - str_map_init (&self->config); + self->config = str_map_make (NULL); simple_config_load_defaults (&self->config, g_config_table); ev_timer_init (&self->quit_timeout_watcher, on_quit_timeout, 3., 0.); self->quit_timeout_watcher.data = self; @@ -1260,9 +1247,7 @@ validate_json_rpc_content_type (const char *content_type) char *type = NULL; char *subtype = NULL; - struct str_map parameters; - str_map_init (¶meters); - parameters.free = free; + struct str_map parameters = str_map_make (free); parameters.key_xfrm = tolower_ascii_strxfrm; bool result = http_parse_media_type @@ -1503,8 +1488,7 @@ request_start (struct request *self, struct str_map *headers) } // Unable to serve the request - struct str response; - str_init (&response); + struct str response = str_make (); str_append (&response, "Status: 404 Not Found\n"); str_append (&response, "Content-Type: text/plain\n\n"); self->write_cb (self->user_data, response.str, response.len); @@ -1536,7 +1520,7 @@ request_handler_json_rpc_try_handle return false; struct str *buf = xcalloc (1, sizeof *buf); - str_init (buf); + *buf = str_make (); request->handler_data = buf; *continue_ = true; @@ -1554,8 +1538,7 @@ request_handler_json_rpc_push return true; } - struct str response; - str_init (&response); + struct str response = str_make (); str_append (&response, "Status: 200 OK\n"); str_append_printf (&response, "Content-Type: %s\n\n", "application/json"); process_json_rpc (request->ctx, buf->str, buf->len, &response); @@ -1587,12 +1570,10 @@ static char * canonicalize_url_path (const char *path) { // XXX: this strips any slashes at the end - struct strv v; - strv_init (&v); + struct strv v = strv_make (); cstr_split (path, "/", true, &v); - struct strv canonical; - strv_init (&canonical); + struct strv canonical = strv_make (); // So that the joined path always begins with a slash strv_append (&canonical, ""); @@ -1672,8 +1653,7 @@ request_handler_static_try_handle FILE *fp = fopen (path, "rb"); if (!fp) { - struct str response; - str_init (&response); + struct str response = str_make (); str_append (&response, "Status: 404 Not Found\n"); str_append (&response, "Content-Type: text/plain\n\n"); str_append_printf (&response, @@ -1699,8 +1679,7 @@ request_handler_static_try_handle if (!mime_type) mime_type = xstrdup ("application/octet_stream"); - struct str response; - str_init (&response); + struct str response = str_make (); str_append (&response, "Status: 200 OK\n"); str_append_printf (&response, "Content-Type: %s\n\n", mime_type); request->write_cb (request->user_data, response.str, response.len); @@ -1878,7 +1857,7 @@ client_init (EV_P_ struct client *self, int sock_fd) memset (self, 0, sizeof *self); self->ctx = ctx; - write_queue_init (&self->write_queue); + self->write_queue = write_queue_make (); set_blocking (sock_fd, false); self->socket_fd = sock_fd; @@ -2119,7 +2098,7 @@ client_scgi_create (EV_P_ int sock_fd) self->request.close_cb = client_scgi_close_cb; self->request.user_data = self; - scgi_parser_init (&self->parser); + self->parser = scgi_parser_make (); self->parser.on_headers_read = client_scgi_on_headers_read; self->parser.on_content = client_scgi_on_content; self->parser.user_data = self; @@ -2145,8 +2124,7 @@ client_ws_on_message (void *user_data, return false; } - struct str response; - str_init (&response); + struct str response = str_make (); process_json_rpc (self->client.ctx, data, len, &response); if (response.len) ws_handler_send (&self->handler, @@ -2410,9 +2388,9 @@ setup_listen_fds (struct server_context *ctx, struct error **e) .ai_flags = AI_PASSIVE, }; - struct strv ports_fcgi; strv_init (&ports_fcgi); - struct strv ports_scgi; strv_init (&ports_scgi); - struct strv ports_ws; strv_init (&ports_ws); + struct strv ports_fcgi = strv_make (); + struct strv ports_scgi = strv_make (); + struct strv ports_ws = strv_make (); get_ports_from_config (ctx, "port_fastcgi", &ports_fcgi); get_ports_from_config (ctx, "port_scgi", &ports_scgi); @@ -2588,8 +2566,8 @@ parse_program_arguments (int argc, char **argv) { 0, NULL, NULL, 0, NULL } }; - struct opt_handler oh; - opt_handler_init (&oh, argc, argv, opts, NULL, "JSON-RPC 2.0 demo server."); + struct opt_handler oh = + opt_handler_make (argc, argv, opts, NULL, "JSON-RPC 2.0 demo server."); int c; while ((c = opt_handler_get (&oh)) != -1) diff --git a/liberty b/liberty index 1dcd259..8ffe20c 160000 --- a/liberty +++ b/liberty @@ -1 +1 @@ -Subproject commit 1dcd259d0506b9e2de3715bdf07144b22f57903a +Subproject commit 8ffe20c0e83b52db1344fe91f57236be4c4cb504 From 8e986a604031aa6fce636a5e1cbb7b1933375406 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Sun, 24 Jun 2018 03:20:55 +0200 Subject: [PATCH 48/79] Remove .travis.yml We don't depend on any proprietary services no longer. I'll have to make my own replacements with blackjack and hookers. Until then, the file stays in the commit log as an example. --- .travis.yml | 27 --------------------------- 1 file changed, 27 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 7395b90..0000000 --- a/.travis.yml +++ /dev/null @@ -1,27 +0,0 @@ -language: c -notifications: - irc: - channels: "irc.janouch.name#dev" - use_notice: true - skip_join: true - template: - - "%{repository_name}#%{build_number} on %{branch}: %{message}" - - " %{compare_url}" - - " %{build_url}" - on_success: change - on_failure: always -compiler: - - clang - - gcc -before_install: - - sudo add-apt-repository ppa:ukplc-team/ppa -y - - sudo apt-get update -qq -install: - - sudo apt-get install -y help2man libjansson-dev libev-dev -before_script: - - mkdir build - - cd build -script: - - cmake .. -DCMAKE_INSTALL_PREFIX=/usr - - make - - cpack -G DEB From df340c13ed99bead31c54bbac003b06af82e7dd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Sun, 24 Jun 2018 05:59:55 +0200 Subject: [PATCH 49/79] Add the missing Date header --- demo-json-rpc-server.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index 0cfb13b..cf4d374 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -925,6 +925,19 @@ ws_handler_http_responsev (struct ws_handler *self, while (*fields) str_append_printf (&response, "%s\r\n", *fields++); + time_t now = time (NULL); + struct tm ts; + gmtime_r (&now, &ts); + + // See RFC 7231, 7.1.1.2. Date + const char *dow[] = { "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat" }; + const char *moy[] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", + "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" }; + str_append_printf (&response, + "Date: %s, %02d %s %04d %02d:%02d:%02d GMT\r\n", + dow[ts.tm_wday], ts.tm_mday, moy[ts.tm_mon], ts.tm_year + 1900, + ts.tm_hour, ts.tm_min, ts.tm_sec); + str_append (&response, "Server: " PROGRAM_NAME "/" PROGRAM_VERSION "\r\n\r\n"); self->write_cb (self->user_data, response.str, response.len); From 711d73f481e7f0254acc3bca3e9dcf9764aaf583 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Sun, 24 Jun 2018 06:02:02 +0200 Subject: [PATCH 50/79] Fix text message UTF-8 validation --- demo-json-rpc-server.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index cf4d374..6ae1742 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -711,7 +711,7 @@ ws_handler_on_frame (void *user_data, const struct ws_parser *parser) return true; if (self->message_opcode == WS_OPCODE_TEXT - && !utf8_validate (self->parser.input.str, self->parser.input.len)) + && !utf8_validate (self->message_data.str, self->message_data.len)) { ws_handler_fail (self, WS_STATUS_INVALID_PAYLOAD_DATA); return false; From 329fc9b88fa419ef2c3a32ccc4414dec223bd69d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Thu, 11 Oct 2018 21:03:34 +0200 Subject: [PATCH 51/79] Bump liberty Eliminates some fall-through warnings. --- liberty | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/liberty b/liberty index 8ffe20c..9494e8e 160000 --- a/liberty +++ b/liberty @@ -1 +1 @@ -Subproject commit 8ffe20c0e83b52db1344fe91f57236be4c4cb504 +Subproject commit 9494e8e2affcd08b791d3ecf68985a8a3a310e55 From d182bcef3bbba9107cb808134d79ffe930be3e14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Fri, 12 Oct 2018 19:59:17 +0200 Subject: [PATCH 52/79] More transient errors --- demo-json-rpc-server.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index 6ae1742..56b931d 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -2285,7 +2285,8 @@ on_client_available (EV_P_ ev_io *watcher, int revents) listener->create (EV_A_ sock_fd); else if (errno == EAGAIN) return; - else if (errno != EINTR && errno != ECONNABORTED) + else if (errno != EINTR && errno != EMFILE + && errno != ECONNRESET && errno != ECONNABORTED) break; } From fd17b4e5046b98b7be7bbc9133140cec2e24777d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Fri, 12 Oct 2018 23:07:06 +0200 Subject: [PATCH 53/79] Update code comments --- demo-json-rpc-server.c | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index 56b931d..995e7e2 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -418,6 +418,7 @@ fcgi_muxer_on_message (const struct fcgi_parser *parser, void *user_data) if (parser->type >= N_ELEMENTS (handlers) || !(handler = handlers[parser->type])) { + // Responding in this way even to application records, unspecified uint8_t content[8] = { parser->type }; fcgi_muxer_send (self, FCGI_UNKNOWN_TYPE, parser->request_id, content, sizeof content); @@ -1303,8 +1304,9 @@ json_rpc_handler_info_cmp (const void *first, const void *second) ((struct json_rpc_handler_info *) second)->method_name); } -// TODO: a method that queues up a ping over IRC: this has to be owned by the -// server context as a background job that removes itself upon completion. +// TODO: a method that sends a response after a certain number of seconds. +// This has to be owned by the server context as a background job that +// removes itself upon completion. static json_t * json_rpc_ping (struct server_context *ctx, json_t *params) @@ -1479,6 +1481,8 @@ request_finish (struct request *self) self->close_cb (self->user_data); } +/// Starts processing a request. Returns false if no further action is to be +/// done and the request should be finished. static bool request_start (struct request *self, struct str_map *headers) { @@ -1489,7 +1493,7 @@ request_start (struct request *self, struct str_map *headers) // // However that might cause some stuff to be done twice. // - // Another way we could get rid off the continue_ argument is via adding + // Another way we could get rid of the continue_ argument is via adding // some way of marking the request as finished from within the handler. bool continue_ = true; @@ -1551,6 +1555,9 @@ request_handler_json_rpc_push return true; } + // TODO: check buf.len against CONTENT_LENGTH; if it's less, then the + // client hasn't been successful in transferring all of its data + struct str response = str_make (); str_append (&response, "Status: 200 OK\n"); str_append_printf (&response, "Content-Type: %s\n\n", "application/json"); @@ -1845,6 +1852,8 @@ static void on_client_ready (EV_P_ ev_io *watcher, int revents) { struct client *client = watcher->data; + // XXX: although read and write are in a sequence, if we create response + // data, we'll still likely need to go back to the event loop. if (revents & EV_READ) if (!client_read_loop (EV_A_ client, watcher)) @@ -2488,6 +2497,9 @@ on_termination_signal (EV_P_ ev_signal *handle, int revents) (void) handle; (void) revents; + // TODO: consider quitting right away if already quitting; + // considering that this may already happen in timeout, it should be OK; + // see on_quit_timeout, just destroy all clients initiate_quit (ctx); } From 8b334e9c9187f4eb8bdbfde7644f82a66d30af00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Sat, 13 Oct 2018 04:08:09 +0200 Subject: [PATCH 54/79] Fix fcgi_muxer_send() Outgoing records were missing padding and the reserved field. --- demo-json-rpc-server.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index 995e7e2..fa7f833 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -145,14 +145,18 @@ fcgi_muxer_send (struct fcgi_muxer *self, hard_assert (len <= UINT16_MAX); struct str message = str_make (); + static char zeroes[8]; + size_t padding = -len & 7; str_pack_u8 (&message, FCGI_VERSION_1); str_pack_u8 (&message, type); str_pack_u16 (&message, request_id); - str_pack_u16 (&message, len); // content length - str_pack_u8 (&message, 0); // padding length + str_pack_u16 (&message, len); // content length + str_pack_u8 (&message, padding); // padding length + str_pack_u8 (&message, 0); // reserved str_append_data (&message, data, len); + str_append_data (&message, zeroes, padding); // XXX: we should probably have another write_cb that assumes ownership self->write_cb (self->user_data, message.str, message.len); From 14ded260a0d8e364f2edb895e915c623e1ab7ce7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Mon, 15 Oct 2018 03:02:49 +0200 Subject: [PATCH 55/79] Clarify and degrade FastCGI multiplexing No need to support more than 255 concurrent requests on one connection. --- demo-json-rpc-server.c | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index fa7f833..aa56563 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -124,10 +124,6 @@ struct fcgi_muxer // TODO: bool quitting; that causes us to reject all requests? - /// Requests assigned to request IDs - // TODO: allocate this dynamically - struct fcgi_request *requests[1 << 16]; - void (*write_cb) (void *user_data, const void *data, size_t len); void (*close_cb) (void *user_data); @@ -136,6 +132,9 @@ struct fcgi_muxer void (*request_destroy_cb) (void *handler_data); void *user_data; ///< User data for callbacks + + /// Requests assigned to request IDs (may not be FCGI_NULL_REQUEST_ID) + struct fcgi_request *requests[1 << 8]; }; static void @@ -284,18 +283,21 @@ fcgi_muxer_on_get_values nv_parser.output = &values; fcgi_nv_parser_push (&nv_parser, parser->content.str, parser->content.len); + const char *key = NULL; - struct str_map_iter iter = str_map_iter_make (&values); - while (str_map_iter_next (&iter)) - { - const char *key = iter.link->key; + // No real-world servers seem to actually use multiplexing + // or even issue this request, but we will implement it anyway + if (str_map_find (&values, (key = FCGI_MPXS_CONNS))) + str_map_set (&response, key, xstrdup ("1")); - // TODO: if (!strcmp (key, FCGI_MAX_CONNS)) - // TODO: if (!strcmp (key, FCGI_MAX_REQS)) + // It's not clear whether FCGI_MAX_REQS means concurrently over all + // connections or over just a single connection (multiplexed), though + // supposedly it's actually per /web server/. Supply the strictest limit. + if (str_map_find (&values, (key = FCGI_MAX_REQS))) + str_map_set (&response, key, + xstrdup_printf ("%zu", N_ELEMENTS (self->requests) - 1)); - if (!strcmp (key, FCGI_MPXS_CONNS)) - str_map_set (&response, key, xstrdup ("1")); - } + // FCGI_MAX_CONNS would be basically infinity. We don't limit connections. struct str content = str_make (); fcgi_nv_convert (&response, &content); From 441c89f654d2f5d8d36c10a948d2ab5db87ffa12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Mon, 15 Oct 2018 03:04:39 +0200 Subject: [PATCH 56/79] Handle FastCGI null request IDs better --- demo-json-rpc-server.c | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index aa56563..096d2a5 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -276,7 +276,14 @@ static void fcgi_muxer_on_get_values (struct fcgi_muxer *self, const struct fcgi_parser *parser) { - struct str_map values = str_map_make (free); + if (parser->request_id != FCGI_NULL_REQUEST_ID) + { + print_debug ("FastCGI: ignoring invalid %s message", + STRINGIFY (FCGI_GET_VALUES)); + return; + } + + struct str_map values = str_map_make (free); struct str_map response = str_map_make (free); struct fcgi_nv_parser nv_parser = fcgi_nv_parser_make (); @@ -330,6 +337,13 @@ fcgi_muxer_on_begin_request return; } + struct fcgi_request *request = self->requests[parser->request_id]; + if (parser->request_id == FCGI_NULL_REQUEST_ID || request) + { + // TODO: fail + return; + } + // We can only act as a responder, reject everything else up front if (role != FCGI_RESPONDER) { @@ -338,10 +352,11 @@ fcgi_muxer_on_begin_request return; } - struct fcgi_request *request = self->requests[parser->request_id]; - if (request) + // TODO: also send OVERLOADED when shutting down? + if (parser->request_id >= N_ELEMENTS (self->requests)) { - // TODO: fail + fcgi_muxer_send_end_request (self, + parser->request_id, 0, FCGI_OVERLOADED); return; } @@ -359,21 +374,22 @@ fcgi_muxer_on_abort_request (struct fcgi_muxer *self, const struct fcgi_parser *parser) { struct fcgi_request *request = self->requests[parser->request_id]; - if (!request) + if (parser->request_id == FCGI_NULL_REQUEST_ID || !request) { print_debug ("FastCGI: received %s for an unknown request", STRINGIFY (FCGI_ABORT_REQUEST)); return; } - // TODO: abort the request: let it somehow produce FCGI_END_REQUEST + // TODO: abort the request: let it somehow produce FCGI_END_REQUEST, + // make sure to send an stdout EOF record } static void fcgi_muxer_on_params (struct fcgi_muxer *self, const struct fcgi_parser *parser) { struct fcgi_request *request = self->requests[parser->request_id]; - if (!request) + if (parser->request_id == FCGI_NULL_REQUEST_ID || !request) { print_debug ("FastCGI: received %s for an unknown request", STRINGIFY (FCGI_PARAMS)); @@ -388,7 +404,7 @@ static void fcgi_muxer_on_stdin (struct fcgi_muxer *self, const struct fcgi_parser *parser) { struct fcgi_request *request = self->requests[parser->request_id]; - if (!request) + if (parser->request_id == FCGI_NULL_REQUEST_ID || !request) { print_debug ("FastCGI: received %s for an unknown request", STRINGIFY (FCGI_STDIN)); From 267a9a561bdc252736a022f241b9de5458d66ceb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Mon, 15 Oct 2018 03:28:09 +0200 Subject: [PATCH 57/79] Eliminate some warnings --- demo-json-rpc-server.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index 096d2a5..3698ade 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -769,7 +769,9 @@ ws_handler_on_ping_timer (EV_P_ ev_timer *watcher, int revents) static void ws_handler_on_close_timeout (EV_P_ ev_timer *watcher, int revents) { + (void) revents; struct ws_handler *self = watcher->data; + // TODO: anything else to do here? Invalidate our state? if (self->close_cb) self->close_cb (self->user_data); @@ -778,6 +780,7 @@ ws_handler_on_close_timeout (EV_P_ ev_timer *watcher, int revents) static void ws_handler_on_handshake_timeout (EV_P_ ev_timer *watcher, int revents) { + (void) revents; struct ws_handler *self = watcher->data; // TODO } @@ -1034,9 +1037,12 @@ ws_handler_finish_handshake (struct ws_handler *self) FAIL_HANDSHAKE (HTTP_400_BAD_REQUEST, NULL); // Okay, we're finally past the basic HTTP/1.1 stuff - const char *key = str_map_find (&self->headers, SEC_WS_KEY); - const char *version = str_map_find (&self->headers, SEC_WS_VERSION); - const char *protocol = str_map_find (&self->headers, SEC_WS_PROTOCOL); + const char *key = str_map_find (&self->headers, SEC_WS_KEY); + const char *version = str_map_find (&self->headers, SEC_WS_VERSION); +/* + const char *protocol = str_map_find (&self->headers, SEC_WS_PROTOCOL); + const char *extensions = str_map_find (&self->headers, SEC_WS_EXTENSIONS); +*/ if (!key) FAIL_HANDSHAKE (HTTP_400_BAD_REQUEST, NULL); @@ -1063,7 +1069,7 @@ ws_handler_finish_handshake (struct ws_handler *self) xstrdup_printf (SEC_WS_ACCEPT ": %s", response_key)); free (response_key); - // TODO: check and set Sec-Websocket-{Extensions,Protocol} + // TODO: make it possible to choose Sec-Websocket-{Extensions,Protocol} ws_handler_http_responsev (self, HTTP_101_SWITCHING_PROTOCOLS, fields.vector); From a14edb72e97084d8c5c16b7c5f37396091464103 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Mon, 15 Oct 2018 05:07:57 +0200 Subject: [PATCH 58/79] Make Doxygen a bit more useful --- demo-json-rpc-server.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index 3698ade..476187c 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -96,6 +96,8 @@ log_message_syslog (void *user_data, const char *quote, const char *fmt, } // --- FastCGI ----------------------------------------------------------------- +/// @defgroup FastCGI +/// @{ enum fcgi_request_state { @@ -470,7 +472,10 @@ fcgi_muxer_push (struct fcgi_muxer *self, const void *data, size_t len) fcgi_parser_push (&self->parser, data, len); } +/// @} // --- WebSockets -------------------------------------------------------------- +/// @defgroup WebSockets +/// @{ // WebSockets aren't CGI-compatible, therefore we must handle the initial HTTP // handshake ourselves. Luckily it's not too much of a bother with http-parser. @@ -1169,6 +1174,7 @@ ws_handler_push (struct ws_handler *self, const void *data, size_t len) return true; } +/// @} // --- Server ------------------------------------------------------------------ static struct simple_config_item g_config_table[] = @@ -1229,6 +1235,8 @@ server_context_free (struct server_context *self) } // --- JSON-RPC ---------------------------------------------------------------- +/// @defgroup JSON-RPC +/// @{ #define JSON_RPC_ERROR_TABLE(XX) \ XX (-32700, PARSE_ERROR, "Parse error") \ @@ -1451,7 +1459,10 @@ process_json_rpc (struct server_context *ctx, } } +/// @} // --- Requests ---------------------------------------------------------------- +/// @defgroup Requests +/// @{ struct request { @@ -1551,6 +1562,7 @@ request_push (struct request *self, const void *data, size_t len) return self->handler->push_cb (self, data, len); } +/// @} // --- Requests handlers ------------------------------------------------------- static bool From 7aff9c3475989278bf0c3bc69e6be499e78d59e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Mon, 15 Oct 2018 19:29:17 +0200 Subject: [PATCH 59/79] Improve documentation --- demo-json-rpc-server.c | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index 476187c..9658e91 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -546,7 +546,7 @@ struct ws_handler // Actually, calling push() could work pretty fine for this. void (*on_close) (void *user_data, int close_code, const char *reason); - // Method callbacks: + // Virtual method callbacks: /// Write a chunk of data to the stream void (*write_cb) (void *user_data, const void *data, size_t len); @@ -1464,11 +1464,12 @@ process_json_rpc (struct server_context *ctx, /// @defgroup Requests /// @{ +/// A generic CGI request abstraction, writing data indirectly through callbacks struct request { struct server_context *ctx; ///< Server context - struct request_handler *handler; ///< Current request handler + struct request_handler *handler; ///< Assigned request handler void *handler_data; ///< User data for the handler /// Callback to write some CGI response data to the output @@ -1481,20 +1482,25 @@ struct request void *user_data; ///< User data argument for callbacks }; +/// An interface to detect and handle specific kinds of CGI requests. +/// The server walks through a list of them until it finds one that can serve +/// a particular request. If unsuccessful, the remote client gets a 404 +/// (the default handling). struct request_handler { LIST_HEADER (struct request_handler) - /// Install ourselves as the handler for the request if applicable. - /// Set @a continue_ to false if further processing should be stopped. + /// Install ourselves as the handler for the request, if applicable. + /// Sets @a continue_ to false if further processing should be stopped, + /// meaning the request has already been handled. bool (*try_handle) (struct request *request, struct str_map *headers, bool *continue_); /// Handle incoming data. - /// Return false if further processing should be stopped. + /// Returns false if there is no more processing to be done. bool (*push_cb) (struct request *request, const void *data, size_t len); - /// Destroy the handler + /// Destroy the handler's data stored in the request object void (*destroy_cb) (struct request *request); }; @@ -1792,6 +1798,8 @@ struct request_handler g_request_handler_static = // --- Client communication handlers ------------------------------------------- +/// A virtual class for client connections coming either from the web server +/// or directly from the end-client, depending on the protocol in use struct client { LIST_HEADER (struct client) @@ -1808,6 +1816,7 @@ struct client struct client_vtable *vtable; ///< Client behaviour }; +/// The concrete behaviour to serve a particular client's requests struct client_vtable { /// Attempt a graceful shutdown From dda22c2cd552d113db6afd789e38c5d436156d7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Mon, 15 Oct 2018 20:38:48 +0200 Subject: [PATCH 60/79] Eliminate unnecessary user_data pointers The CONTAINER_OF macro can find the parent structure just as well. --- demo-json-rpc-server.c | 122 +++++++++++++++++++---------------------- 1 file changed, 57 insertions(+), 65 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index 9658e91..baed884 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -44,6 +44,9 @@ enum { PIPE_READ, PIPE_WRITE }; +#define FIND_CONTAINER(name, pointer, type, member) \ + type *name = CONTAINER_OF (pointer, type, member) + // --- libev helpers ----------------------------------------------------------- static bool @@ -126,15 +129,13 @@ struct fcgi_muxer // TODO: bool quitting; that causes us to reject all requests? - void (*write_cb) (void *user_data, const void *data, size_t len); - void (*close_cb) (void *user_data); + void (*write_cb) (struct fcgi_muxer *, const void *data, size_t len); + void (*close_cb) (struct fcgi_muxer *); - void *(*request_start_cb) (void *user_data, struct fcgi_request *request); + void *(*request_start_cb) (struct fcgi_muxer *, struct fcgi_request *); void (*request_push_cb) (void *handler_data, const void *data, size_t len); void (*request_destroy_cb) (void *handler_data); - void *user_data; ///< User data for callbacks - /// Requests assigned to request IDs (may not be FCGI_NULL_REQUEST_ID) struct fcgi_request *requests[1 << 8]; }; @@ -160,7 +161,7 @@ fcgi_muxer_send (struct fcgi_muxer *self, str_append_data (&message, zeroes, padding); // XXX: we should probably have another write_cb that assumes ownership - self->write_cb (self->user_data, message.str, message.len); + self->write_cb (self, message.str, message.len); str_free (&message); } @@ -210,8 +211,7 @@ fcgi_request_push_params { // TODO: probably check the state of the header parser // TODO: request_start() can return false, end the request here? - self->handler_data = self->muxer->request_start_cb - (self->muxer->user_data, self); + self->handler_data = self->muxer->request_start_cb (self->muxer, self); self->state = FCGI_REQUEST_STDIN; } } @@ -532,10 +532,10 @@ struct ws_handler // This may render "on_connected" unnecessary. /// Called after successfuly connecting (handshake complete) - bool (*on_connected) (void *user_data); + bool (*on_connected) (struct ws_handler *); /// Called upon reception of a single full message - bool (*on_message) (void *user_data, + bool (*on_message) (struct ws_handler *, enum ws_opcode type, const void *data, size_t len); /// The connection is about to close. @a close_code may, or may not, be one @@ -544,17 +544,15 @@ struct ws_handler // receive a notification about the connection being closed because of // an error (recv()) returns -1, and call on_close() in reaction. // Actually, calling push() could work pretty fine for this. - void (*on_close) (void *user_data, int close_code, const char *reason); + void (*on_close) (struct ws_handler *, int close_code, const char *reason); // Virtual method callbacks: /// Write a chunk of data to the stream - void (*write_cb) (void *user_data, const void *data, size_t len); + void (*write_cb) (struct ws_handler *, const void *data, size_t len); /// Close the connection - void (*close_cb) (void *user_data); - - void *user_data; ///< User data for callbacks + void (*close_cb) (struct ws_handler *); }; static void @@ -569,8 +567,8 @@ ws_handler_send_control (struct ws_handler *self, } uint8_t header[2] = { 0x80 | (opcode & 0x0F), len }; - self->write_cb (self->user_data, header, sizeof header); - self->write_cb (self->user_data, data, len); + self->write_cb (self, header, sizeof header); + self->write_cb (self, data, len); } static void @@ -585,7 +583,7 @@ ws_handler_close (struct ws_handler *self, // Close initiated by us; the reason is null-terminated within `payload' if (self->on_close) - self->on_close (self->user_data, close_code, payload.str + 2); + self->on_close (self, close_code, payload.str + 2); self->state = WS_HANDLER_CLOSING; str_free (&payload); @@ -628,8 +626,8 @@ ws_handler_send (struct ws_handler *self, else str_pack_u8 (&header, len); - self->write_cb (self->user_data, header.str, header.len); - self->write_cb (self->user_data, data, len); + self->write_cb (self, header.str, header.len); + self->write_cb (self, data, len); str_free (&header); } @@ -680,7 +678,7 @@ ws_handler_on_protocol_close ws_handler_send_control (self, WS_OPCODE_CLOSE, parser->input.str, parser->payload_len); if (self->on_close) - self->on_close (self->user_data, close_code, reason); + self->on_close (self, close_code, reason); } free (reason); @@ -747,7 +745,7 @@ ws_handler_on_frame (void *user_data, const struct ws_parser *parser) bool result = true; if (self->on_message) - result = self->on_message (self->user_data, self->message_opcode, + result = self->on_message (self, self->message_opcode, self->message_data.str, self->message_data.len); str_reset (&self->message_data); return result; @@ -779,7 +777,7 @@ ws_handler_on_close_timeout (EV_P_ ev_timer *watcher, int revents) // TODO: anything else to do here? Invalidate our state? if (self->close_cb) - self->close_cb (self->user_data); + self->close_cb (self); } static void @@ -971,7 +969,7 @@ ws_handler_http_responsev (struct ws_handler *self, str_append (&response, "Server: " PROGRAM_NAME "/" PROGRAM_VERSION "\r\n\r\n"); - self->write_cb (self->user_data, response.str, response.len); + self->write_cb (self, response.str, response.len); str_free (&response); } @@ -1110,8 +1108,7 @@ ws_handler_push (struct ws_handler *self, const void *data, size_t len) if (self->state == WS_HANDLER_OPEN) { if (self->on_close) - self->on_close (self->user_data, - WS_STATUS_ABNORMAL_CLOSURE, ""); + self->on_close (self, WS_STATUS_ABNORMAL_CLOSURE, ""); } else { @@ -1154,7 +1151,7 @@ ws_handler_push (struct ws_handler *self, const void *data, size_t len) self->state = WS_HANDLER_OPEN; if (self->on_connected) - return self->on_connected (self->user_data); + return self->on_connected (self); return true; } @@ -1473,13 +1470,11 @@ struct request void *handler_data; ///< User data for the handler /// Callback to write some CGI response data to the output - void (*write_cb) (void *user_data, const void *data, size_t len); + void (*write_cb) (struct request *, const void *data, size_t len); /// Callback to close the connection. /// CALLING THIS MAY CAUSE THE REQUEST TO BE DESTROYED. - void (*close_cb) (void *user_data); - - void *user_data; ///< User data argument for callbacks + void (*close_cb) (struct request *); }; /// An interface to detect and handle specific kinds of CGI requests. @@ -1523,7 +1518,7 @@ request_free (struct request *self) static void request_finish (struct request *self) { - self->close_cb (self->user_data); + self->close_cb (self); } /// Starts processing a request. Returns false if no further action is to be @@ -1553,7 +1548,7 @@ request_start (struct request *self, struct str_map *headers) struct str response = str_make (); str_append (&response, "Status: 404 Not Found\n"); str_append (&response, "Content-Type: text/plain\n\n"); - self->write_cb (self->user_data, response.str, response.len); + self->write_cb (self, response.str, response.len); str_free (&response); return false; } @@ -1608,7 +1603,7 @@ request_handler_json_rpc_push str_append (&response, "Status: 200 OK\n"); str_append_printf (&response, "Content-Type: %s\n\n", "application/json"); process_json_rpc (request->ctx, buf->str, buf->len, &response); - request->write_cb (request->user_data, response.str, response.len); + request->write_cb (request, response.str, response.len); str_free (&response); return false; } @@ -1724,7 +1719,7 @@ request_handler_static_try_handle str_append (&response, "Content-Type: text/plain\n\n"); str_append_printf (&response, "File %s was not found on this server\n", suffix); - request->write_cb (request->user_data, response.str, response.len); + request->write_cb (request, response.str, response.len); str_free (&response); free (suffix); @@ -1748,17 +1743,17 @@ request_handler_static_try_handle struct str response = str_make (); str_append (&response, "Status: 200 OK\n"); str_append_printf (&response, "Content-Type: %s\n\n", mime_type); - request->write_cb (request->user_data, response.str, response.len); + request->write_cb (request, response.str, response.len); str_free (&response); free (mime_type); // Write the chunk we've used to help us with magic detection; // obviously we have to do it after we've written the headers if (len) - request->write_cb (request->user_data, buf, len); + request->write_cb (request, buf, len); while ((len = fread (buf, 1, sizeof buf, fp))) - request->write_cb (request->user_data, buf, len); + request->write_cb (request, buf, len); fclose (fp); // TODO: this should rather not be returned all at once but in chunks; @@ -1962,28 +1957,29 @@ struct client_fcgi_request // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - static void -client_fcgi_request_write_cb (void *user_data, const void *data, size_t len) +client_fcgi_request_write_cb (struct request *req, const void *data, size_t len) { - struct client_fcgi_request *request = user_data; - fcgi_request_write (request->fcgi_request, data, len); + FIND_CONTAINER (self, req, struct client_fcgi_request, request); + fcgi_request_write (self->fcgi_request, data, len); } static void -client_fcgi_request_close_cb (void *user_data) +client_fcgi_request_close_cb (struct request *req) { - struct client_fcgi_request *request = user_data; + FIND_CONTAINER (self, req, struct client_fcgi_request, request); // No more data to send, terminate the substream/request // XXX: this will most probably end up with client_fcgi_request_destroy(), // we might or might not need to defer this action - fcgi_request_finish (request->fcgi_request); + fcgi_request_finish (self->fcgi_request); } // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - static void * -client_fcgi_request_start (void *user_data, struct fcgi_request *fcgi_request) +client_fcgi_request_start + (struct fcgi_muxer *mux, struct fcgi_request *fcgi_request) { - struct client_fcgi *self = user_data; + FIND_CONTAINER (self, mux, struct client_fcgi, muxer); // TODO: what if the request is aborted by ; struct client_fcgi_request *request = xcalloc (1, sizeof *request); @@ -1992,7 +1988,6 @@ client_fcgi_request_start (void *user_data, struct fcgi_request *fcgi_request) request->request.ctx = self->client.ctx; request->request.write_cb = client_fcgi_request_write_cb; request->request.close_cb = client_fcgi_request_close_cb; - request->request.user_data = request; return request; } @@ -2014,16 +2009,16 @@ client_fcgi_request_destroy (void *handler_data) // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - static void -client_fcgi_write_cb (void *user_data, const void *data, size_t len) +client_fcgi_write_cb (struct fcgi_muxer *mux, const void *data, size_t len) { - struct client_fcgi *self = user_data; + FIND_CONTAINER (self, mux, struct client_fcgi, muxer); client_write (&self->client, data, len); } static void -client_fcgi_close_cb (void *user_data) +client_fcgi_close_cb (struct fcgi_muxer *mux) { - struct client_fcgi *self = user_data; + FIND_CONTAINER (self, mux, struct client_fcgi, muxer); // FIXME: we should probably call something like client_shutdown(), // which may have an argument whether we should really use close() client_destroy (&self->client); @@ -2076,7 +2071,6 @@ client_fcgi_create (EV_P_ int sock_fd) self->muxer.request_start_cb = client_fcgi_request_start; self->muxer.request_push_cb = client_fcgi_request_push; self->muxer.request_destroy_cb = client_fcgi_request_destroy; - self->muxer.user_data = self; return &self->client; } @@ -2090,17 +2084,17 @@ struct client_scgi }; static void -client_scgi_write_cb (void *user_data, const void *data, size_t len) +client_scgi_write_cb (struct request *req, const void *data, size_t len) { - struct client_scgi *self = user_data; + FIND_CONTAINER (self, req, struct client_scgi, request); client_write (&self->client, data, len); } static void -client_scgi_close_cb (void *user_data) +client_scgi_close_cb (struct request *req) { // NOTE: this rather really means "close me [the request]" - struct client_scgi *self = user_data; + FIND_CONTAINER (self, req, struct client_scgi, request); // FIXME: we should probably call something like client_shutdown(), // which may have an argument whether we should really use close() client_destroy (&self->client); @@ -2167,7 +2161,6 @@ client_scgi_create (EV_P_ int sock_fd) self->request.ctx = self->client.ctx; self->request.write_cb = client_scgi_write_cb; self->request.close_cb = client_scgi_close_cb; - self->request.user_data = self; self->parser = scgi_parser_make (); self->parser.on_headers_read = client_scgi_on_headers_read; @@ -2185,10 +2178,10 @@ struct client_ws }; static bool -client_ws_on_message (void *user_data, +client_ws_on_message (struct ws_handler *handler, enum ws_opcode type, const void *data, size_t len) { - struct client_ws *self = user_data; + FIND_CONTAINER (self, handler, struct client_ws, handler); if (type != WS_OPCODE_TEXT) { ws_handler_fail (&self->handler, WS_STATUS_UNSUPPORTED_DATA); @@ -2205,16 +2198,16 @@ client_ws_on_message (void *user_data, } static void -client_ws_write_cb (void *user_data, const void *data, size_t len) +client_ws_write_cb (struct ws_handler *handler, const void *data, size_t len) { - struct client *client = user_data; - client_write (client, data, len); + FIND_CONTAINER (self, handler, struct client_ws, handler); + client_write (&self->client, data, len); } static void -client_ws_close_cb (void *user_data) +client_ws_close_cb (struct ws_handler *handler) { - struct client_ws *self = user_data; + FIND_CONTAINER (self, handler, struct client_ws, handler); // FIXME: we should probably call something like client_shutdown(), // which may have an argument whether we should really use close() client_destroy (&self->client); @@ -2261,7 +2254,6 @@ client_ws_create (EV_P_ int sock_fd) self->handler.on_message = client_ws_on_message; self->handler.write_cb = client_ws_write_cb; self->handler.close_cb = client_ws_close_cb; - self->handler.user_data = self; // One mebibyte seems to be a reasonable value self->handler.max_payload_len = 1 << 10; From e9530f450e64947850da0ed7596c8611cf445efe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Tue, 16 Oct 2018 00:18:13 +0200 Subject: [PATCH 61/79] Call ws_handler_start() --- demo-json-rpc-server.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index baed884..2c487f5 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -2257,6 +2257,8 @@ client_ws_create (EV_P_ int sock_fd) // One mebibyte seems to be a reasonable value self->handler.max_payload_len = 1 << 10; + + ws_handler_start (&self->handler); return &self->client; } From 7cefdd496f8e45a5d4ee54b7c6cfc5283ae32d51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Tue, 16 Oct 2018 04:05:42 +0200 Subject: [PATCH 62/79] Cleanup --- demo-json-rpc-server.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index 2c487f5..4ec7e4c 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -2218,21 +2218,21 @@ client_ws_close_cb (struct ws_handler *handler) static void client_ws_shutdown (struct client *client) { - struct client_ws *self = (struct client_ws *) client; + FIND_CONTAINER (self, client, struct client_ws, client); ws_handler_close (&self->handler, WS_STATUS_GOING_AWAY, NULL, 0); } static void client_ws_destroy (struct client *client) { - struct client_ws *self = (struct client_ws *) client; + FIND_CONTAINER (self, client, struct client_ws, client); ws_handler_free (&self->handler); } static bool client_ws_push (struct client *client, const void *data, size_t len) { - struct client_ws *self = (struct client_ws *) client; + FIND_CONTAINER (self, client, struct client_ws, client); return ws_handler_push (&self->handler, data, len); } From 7f6db9d39f7a65243992b3a01eacedb4b2be279e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Tue, 16 Oct 2018 04:09:59 +0200 Subject: [PATCH 63/79] Improve WebSocket shutdown --- demo-json-rpc-server.c | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index 4ec7e4c..4bf1fcc 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -540,10 +540,6 @@ struct ws_handler /// The connection is about to close. @a close_code may, or may not, be one /// of enum ws_status. The @a reason is never NULL. - // TODO; also note that ideally, the handler should (be able to) first - // receive a notification about the connection being closed because of - // an error (recv()) returns -1, and call on_close() in reaction. - // Actually, calling push() could work pretty fine for this. void (*on_close) (struct ws_handler *, int close_code, const char *reason); // Virtual method callbacks: @@ -551,8 +547,9 @@ struct ws_handler /// Write a chunk of data to the stream void (*write_cb) (struct ws_handler *, const void *data, size_t len); - /// Close the connection - void (*close_cb) (struct ws_handler *); + /// Close the connection. If @a half_close is false, you are allowed to + /// destroy the handler directly from within the callback. + void (*close_cb) (struct ws_handler *, bool half_close); }; static void @@ -777,7 +774,7 @@ ws_handler_on_close_timeout (EV_P_ ev_timer *watcher, int revents) // TODO: anything else to do here? Invalidate our state? if (self->close_cb) - self->close_cb (self); + self->close_cb (self, false /* half_close */); } static void @@ -2205,12 +2202,13 @@ client_ws_write_cb (struct ws_handler *handler, const void *data, size_t len) } static void -client_ws_close_cb (struct ws_handler *handler) +client_ws_close_cb (struct ws_handler *handler, bool half_close) { FIND_CONTAINER (self, handler, struct client_ws, handler); - // FIXME: we should probably call something like client_shutdown(), - // which may have an argument whether we should really use close() - client_destroy (&self->client); + if (half_close) + ; // FIXME: we should probably call something like client_shutdown() + else + client_destroy (&self->client); } // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -2219,7 +2217,10 @@ static void client_ws_shutdown (struct client *client) { FIND_CONTAINER (self, client, struct client_ws, client); - ws_handler_close (&self->handler, WS_STATUS_GOING_AWAY, NULL, 0); + if (self->handler.state == WS_HANDLER_CONNECTING) + ; // TODO: abort the connection immediately + else if (self->handler.state == WS_HANDLER_OPEN) + ws_handler_close (&self->handler, WS_STATUS_GOING_AWAY, NULL, 0); } static void From 7d922352ead78b92247d0848572a7e9c50ae1075 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Tue, 16 Oct 2018 04:12:37 +0200 Subject: [PATCH 64/79] Rename client_vtable::destroy to finalize Matches a similar concept from garbage-collected languages. --- demo-json-rpc-server.c | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index 4bf1fcc..f241030 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -1814,9 +1814,8 @@ struct client_vtable /// Attempt a graceful shutdown void (*shutdown) (struct client *client); - /// Do any additional cleanup - // TODO: rename to "finalize" or "cleanup"? - void (*destroy) (struct client *client); + /// Do any additional cleanup for the concrete class before destruction + void (*finalize) (struct client *client); /// Process incoming data; "len == 0" means EOF bool (*push) (struct client *client, const void *data, size_t len); @@ -1848,7 +1847,7 @@ client_destroy (struct client *self) ctx->n_clients--; // First uninitialize the higher-level implementation - self->vtable->destroy (self); + self->vtable->finalize (self); ev_io_stop (EV_DEFAULT_ &self->read_watcher); ev_io_stop (EV_DEFAULT_ &self->write_watcher); @@ -2034,7 +2033,7 @@ client_fcgi_shutdown (struct client *client) } static void -client_fcgi_destroy (struct client *client) +client_fcgi_finalize (struct client *client) { struct client_fcgi *self = (struct client_fcgi *) client; fcgi_muxer_free (&self->muxer); @@ -2051,7 +2050,7 @@ client_fcgi_push (struct client *client, const void *data, size_t len) static struct client_vtable client_fcgi_vtable = { .shutdown = client_fcgi_shutdown, - .destroy = client_fcgi_destroy, + .finalize = client_fcgi_finalize, .push = client_fcgi_push, }; @@ -2118,7 +2117,7 @@ client_scgi_on_content (void *user_data, const void *data, size_t len) // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - static void -client_scgi_destroy (struct client *client) +client_scgi_finalize (struct client *client) { struct client_scgi *self = (struct client_scgi *) client; request_free (&self->request); @@ -2143,8 +2142,8 @@ client_scgi_push (struct client *client, const void *data, size_t len) static struct client_vtable client_scgi_vtable = { - .destroy = client_scgi_destroy, - .push = client_scgi_push, + .finalize = client_scgi_finalize, + .push = client_scgi_push, }; static struct client * @@ -2224,7 +2223,7 @@ client_ws_shutdown (struct client *client) } static void -client_ws_destroy (struct client *client) +client_ws_finalize (struct client *client) { FIND_CONTAINER (self, client, struct client_ws, client); ws_handler_free (&self->handler); @@ -2240,7 +2239,7 @@ client_ws_push (struct client *client, const void *data, size_t len) static struct client_vtable client_ws_vtable = { .shutdown = client_ws_shutdown, - .destroy = client_ws_destroy, + .finalize = client_ws_finalize, .push = client_ws_push, }; From 4c54bc42b9fdb83d439dc974953f06f5def3ae5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Tue, 16 Oct 2018 04:33:33 +0200 Subject: [PATCH 65/79] Clean up and better document client_vtable --- demo-json-rpc-server.c | 65 +++++++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 30 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index f241030..d461691 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -1811,14 +1811,19 @@ struct client /// The concrete behaviour to serve a particular client's requests struct client_vtable { - /// Attempt a graceful shutdown + /// Process incoming data; "len == 0" means EOF. + /// If the method returns false, the client is destroyed by caller. + bool (*push) (struct client *client, const void *data, size_t len); + + // TODO: optional push_error() to inform about network I/O errors + + /// Attempt a graceful shutdown: make any appropriate steps before + /// the client connection times out and gets torn down by force. + /// The client is allowed to destroy itself immediately. void (*shutdown) (struct client *client); /// Do any additional cleanup for the concrete class before destruction void (*finalize) (struct client *client); - - /// Process incoming data; "len == 0" means EOF - bool (*push) (struct client *client, const void *data, size_t len); }; static void @@ -2022,6 +2027,14 @@ client_fcgi_close_cb (struct fcgi_muxer *mux) // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +static bool +client_fcgi_push (struct client *client, const void *data, size_t len) +{ + struct client_fcgi *self = (struct client_fcgi *) client; + fcgi_muxer_push (&self->muxer, data, len); + return true; +} + static void client_fcgi_shutdown (struct client *client) { @@ -2039,19 +2052,11 @@ client_fcgi_finalize (struct client *client) fcgi_muxer_free (&self->muxer); } -static bool -client_fcgi_push (struct client *client, const void *data, size_t len) -{ - struct client_fcgi *self = (struct client_fcgi *) client; - fcgi_muxer_push (&self->muxer, data, len); - return true; -} - static struct client_vtable client_fcgi_vtable = { + .push = client_fcgi_push, .shutdown = client_fcgi_shutdown, .finalize = client_fcgi_finalize, - .push = client_fcgi_push, }; static struct client * @@ -2116,14 +2121,6 @@ client_scgi_on_content (void *user_data, const void *data, size_t len) // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -static void -client_scgi_finalize (struct client *client) -{ - struct client_scgi *self = (struct client_scgi *) client; - request_free (&self->request); - scgi_parser_free (&self->parser); -} - static bool client_scgi_push (struct client *client, const void *data, size_t len) { @@ -2140,10 +2137,18 @@ client_scgi_push (struct client *client, const void *data, size_t len) return false; } +static void +client_scgi_finalize (struct client *client) +{ + struct client_scgi *self = (struct client_scgi *) client; + request_free (&self->request); + scgi_parser_free (&self->parser); +} + static struct client_vtable client_scgi_vtable = { - .finalize = client_scgi_finalize, .push = client_scgi_push, + .finalize = client_scgi_finalize, }; static struct client * @@ -2212,6 +2217,13 @@ client_ws_close_cb (struct ws_handler *handler, bool half_close) // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +static bool +client_ws_push (struct client *client, const void *data, size_t len) +{ + FIND_CONTAINER (self, client, struct client_ws, client); + return ws_handler_push (&self->handler, data, len); +} + static void client_ws_shutdown (struct client *client) { @@ -2229,18 +2241,11 @@ client_ws_finalize (struct client *client) ws_handler_free (&self->handler); } -static bool -client_ws_push (struct client *client, const void *data, size_t len) -{ - FIND_CONTAINER (self, client, struct client_ws, client); - return ws_handler_push (&self->handler, data, len); -} - static struct client_vtable client_ws_vtable = { + .push = client_ws_push, .shutdown = client_ws_shutdown, .finalize = client_ws_finalize, - .push = client_ws_push, }; static struct client * From 272145ace2d7d4bd0342cce1890410d20d861e37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Wed, 17 Oct 2018 02:21:19 +0200 Subject: [PATCH 66/79] Clarify EOF behaviour --- demo-json-rpc-server.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index d461691..c886364 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -413,6 +413,7 @@ fcgi_muxer_on_stdin (struct fcgi_muxer *self, const struct fcgi_parser *parser) return; } + // At the end of the stream, a zero-length record is received fcgi_request_push_stdin (request, parser->content.str, parser->content.len); } @@ -1092,7 +1093,7 @@ ws_handler_start (struct ws_handler *self) ev_timer_start (EV_DEFAULT_ &self->handshake_timeout_watcher); } -/// Push data to the WebSocket handler; "len == 0" means EOF +/// Push data to the WebSocket handler. "len == 0" means EOF. static bool ws_handler_push (struct ws_handler *self, const void *data, size_t len) { @@ -1488,7 +1489,7 @@ struct request_handler bool (*try_handle) (struct request *request, struct str_map *headers, bool *continue_); - /// Handle incoming data. + /// Handle incoming data. "len == 0" means EOF. /// Returns false if there is no more processing to be done. bool (*push_cb) (struct request *request, const void *data, size_t len); @@ -1768,10 +1769,10 @@ request_handler_static_push { (void) request; (void) data; - (void) len; - // Ignoring all content; we shouldn't receive any (GET) - return false; + // Aborting on content; we shouldn't receive any (GET) + // FIXME: there should at least be some indication of this happening + return len == 0; } static void From 83363e6383a7589d2a691ca1f988acb38ccea5b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Wed, 17 Oct 2018 02:57:08 +0200 Subject: [PATCH 67/79] FastCGI: make it work at least in theory --- demo-json-rpc-server.c | 104 +++++++++++++++++++++++++++++------------ 1 file changed, 75 insertions(+), 29 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index c886364..a915b3d 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -123,18 +123,32 @@ struct fcgi_request void *handler_data; ///< Handler data }; +/// Handles a single FastCGI connection, de/multiplexing requests and responses struct fcgi_muxer { struct fcgi_parser parser; ///< FastCGI message parser + uint32_t active_requests; ///< The number of active requests + bool in_shutdown; ///< Rejecting new requests - // TODO: bool quitting; that causes us to reject all requests? + // Virtual method callbacks: + /// Write data to the underlying transport void (*write_cb) (struct fcgi_muxer *, const void *data, size_t len); + + /// Close the underlying transport + // TODO: consider half-close and the subsequent handling void (*close_cb) (struct fcgi_muxer *); - void *(*request_start_cb) (struct fcgi_muxer *, struct fcgi_request *); - void (*request_push_cb) (void *handler_data, const void *data, size_t len); - void (*request_destroy_cb) (void *handler_data); + /// Start processing a request. Return false if no further action is + /// to be done and the request should be finished. + bool (*request_start_cb) (struct fcgi_muxer *, struct fcgi_request *); + + /// Handle incoming data. "len == 0" means EOF. + void (*request_push_cb) + (struct fcgi_request *, const void *data, size_t len); + + /// Destroy the handler's data stored in the request object + void (*request_destroy_cb) (struct fcgi_request *); /// Requests assigned to request IDs (may not be FCGI_NULL_REQUEST_ID) struct fcgi_request *requests[1 << 8]; @@ -177,22 +191,27 @@ fcgi_muxer_send_end_request (struct fcgi_muxer *self, uint16_t request_id, // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -static void -fcgi_request_init (struct fcgi_request *self) +static struct fcgi_request * +fcgi_request_new (void) { - memset (self, 0, sizeof *self); + struct fcgi_request *self = xcalloc (1, sizeof *self); self->headers = str_map_make (free); self->hdr_parser = fcgi_nv_parser_make (); self->hdr_parser.output = &self->headers; + return self; } static void -fcgi_request_free (struct fcgi_request *self) +fcgi_request_destroy (struct fcgi_request *self) { + // TODO: consider the case where it hasn't been started yet + self->muxer->request_destroy_cb (self); + str_map_free (&self->headers); fcgi_nv_parser_free (&self->hdr_parser); + free (self); } static void @@ -211,7 +230,7 @@ fcgi_request_push_params { // TODO: probably check the state of the header parser // TODO: request_start() can return false, end the request here? - self->handler_data = self->muxer->request_start_cb (self->muxer, self); + (void) self->muxer->request_start_cb (self->muxer, self); self->state = FCGI_REQUEST_STDIN; } } @@ -226,7 +245,7 @@ fcgi_request_push_stdin return; } - self->muxer->request_push_cb (self->handler_data, data, len); + self->muxer->request_push_cb (self, data, len); } static void @@ -266,7 +285,21 @@ fcgi_request_write (struct fcgi_request *self, const void *data, size_t len) static void fcgi_request_finish (struct fcgi_request *self) { - // TODO: flush(), end_request(), delete self, muxer->request_destroy_cb()? + fcgi_request_flush (self); + fcgi_muxer_send (self->muxer, FCGI_STDOUT, self->request_id, NULL, 0); + + fcgi_muxer_send_end_request (self->muxer, self->request_id, + 0 /* TODO app_status, although ignored */, + FCGI_REQUEST_COMPLETE /* TODO protocol_status, may be different */); + + if (!(self->flags & FCGI_KEEP_CONN)) + { + // TODO: tear down (shut down) the connection + } + + self->muxer->active_requests--; + self->muxer->requests[self->request_id] = NULL; + fcgi_request_destroy (self); } // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -354,21 +387,21 @@ fcgi_muxer_on_begin_request return; } - // TODO: also send OVERLOADED when shutting down? - if (parser->request_id >= N_ELEMENTS (self->requests)) + if (parser->request_id >= N_ELEMENTS (self->requests) + || self->in_shutdown) { fcgi_muxer_send_end_request (self, parser->request_id, 0, FCGI_OVERLOADED); return; } - request = xcalloc (1, sizeof *request); - fcgi_request_init (request); + request = fcgi_request_new (); request->muxer = self; request->request_id = parser->request_id; request->flags = flags; self->requests[parser->request_id] = request; + self->active_requests++; } static void @@ -464,6 +497,17 @@ fcgi_muxer_init (struct fcgi_muxer *self) static void fcgi_muxer_free (struct fcgi_muxer *self) { + for (size_t i = 0; i < N_ELEMENTS (self->requests); i++) + { + if (!self->active_requests) + break; + if (self->requests[i]) + { + fcgi_request_destroy (self->requests[i]); + self->active_requests--; + } + } + fcgi_parser_free (&self->parser); } @@ -1977,33 +2021,35 @@ client_fcgi_request_close_cb (struct request *req) // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -static void * +static bool client_fcgi_request_start (struct fcgi_muxer *mux, struct fcgi_request *fcgi_request) { FIND_CONTAINER (self, mux, struct client_fcgi, muxer); - // TODO: what if the request is aborted by ; - struct client_fcgi_request *request = xcalloc (1, sizeof *request); + struct client_fcgi_request *request = + fcgi_request->handler_data = xcalloc (1, sizeof *request); request->fcgi_request = fcgi_request; request_init (&request->request); request->request.ctx = self->client.ctx; request->request.write_cb = client_fcgi_request_write_cb; request->request.close_cb = client_fcgi_request_close_cb; - return request; + + return request_start (&request->request, &fcgi_request->headers); } static void -client_fcgi_request_push (void *handler_data, const void *data, size_t len) +client_fcgi_request_push + (struct fcgi_request *req, const void *data, size_t len) { - struct client_fcgi_request *request = handler_data; + struct client_fcgi_request *request = req->handler_data; request_push (&request->request, data, len); } static void -client_fcgi_request_destroy (void *handler_data) +client_fcgi_request_destroy (struct fcgi_request *req) { - struct client_fcgi_request *request = handler_data; + struct client_fcgi_request *request = req->handler_data; request_free (&request->request); free (request); } @@ -2031,7 +2077,7 @@ client_fcgi_close_cb (struct fcgi_muxer *mux) static bool client_fcgi_push (struct client *client, const void *data, size_t len) { - struct client_fcgi *self = (struct client_fcgi *) client; + FIND_CONTAINER (self, client, struct client_fcgi, client); fcgi_muxer_push (&self->muxer, data, len); return true; } @@ -2039,17 +2085,17 @@ client_fcgi_push (struct client *client, const void *data, size_t len) static void client_fcgi_shutdown (struct client *client) { - struct client_fcgi *self = (struct client_fcgi *) client; + FIND_CONTAINER (self, client, struct client_fcgi, client); + self->muxer.in_shutdown = true; - // TODO: respond with FCGI_END_REQUEST: FCGI_REQUEST_COMPLETE to everything, - // and start sending out FCGI_OVERLOADED to all incoming requests. The - // FastCGI specification isn't very clear about what we should do. + // TODO: respond with FCGI_END_REQUEST: FCGI_REQUEST_COMPLETE to everything? + // The FastCGI specification isn't very clear about what we should do. } static void client_fcgi_finalize (struct client *client) { - struct client_fcgi *self = (struct client_fcgi *) client; + FIND_CONTAINER (self, client, struct client_fcgi, client); fcgi_muxer_free (&self->muxer); } From 1d638c9170c2f785000aa1e7fff30974ae9be3d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Wed, 17 Oct 2018 03:53:07 +0200 Subject: [PATCH 68/79] Say "finalize" instead of "destroy" where appropriate - _make() returns a struct directly - _init() initializes over a pointer - _free() deinitializes over a pointer - _new() is like _init() but also allocates - _destroy() is like _free() but also deallocates Finalization is a matching concept in garbage-collected languages. --- demo-json-rpc-server.c | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index a915b3d..8ad1c36 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -148,7 +148,7 @@ struct fcgi_muxer (struct fcgi_request *, const void *data, size_t len); /// Destroy the handler's data stored in the request object - void (*request_destroy_cb) (struct fcgi_request *); + void (*request_finalize_cb) (struct fcgi_request *); /// Requests assigned to request IDs (may not be FCGI_NULL_REQUEST_ID) struct fcgi_request *requests[1 << 8]; @@ -207,7 +207,7 @@ static void fcgi_request_destroy (struct fcgi_request *self) { // TODO: consider the case where it hasn't been started yet - self->muxer->request_destroy_cb (self); + self->muxer->request_finalize_cb (self); str_map_free (&self->headers); fcgi_nv_parser_free (&self->hdr_parser); @@ -1538,7 +1538,7 @@ struct request_handler bool (*push_cb) (struct request *request, const void *data, size_t len); /// Destroy the handler's data stored in the request object - void (*destroy_cb) (struct request *request); + void (*finalize_cb) (struct request *request); }; static void @@ -1551,7 +1551,7 @@ static void request_free (struct request *self) { if (self->handler) - self->handler->destroy_cb (self); + self->handler->finalize_cb (self); } /// This function is only intended to be run from asynchronous event handlers @@ -1651,7 +1651,7 @@ request_handler_json_rpc_push } static void -request_handler_json_rpc_destroy (struct request *request) +request_handler_json_rpc_finalize (struct request *request) { struct str *buf = request->handler_data; str_free (buf); @@ -1662,9 +1662,9 @@ request_handler_json_rpc_destroy (struct request *request) struct request_handler g_request_handler_json_rpc = { - .try_handle = request_handler_json_rpc_try_handle, - .push_cb = request_handler_json_rpc_push, - .destroy_cb = request_handler_json_rpc_destroy, + .try_handle = request_handler_json_rpc_try_handle, + .push_cb = request_handler_json_rpc_push, + .finalize_cb = request_handler_json_rpc_finalize, }; // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -1820,7 +1820,7 @@ request_handler_static_push } static void -request_handler_static_destroy (struct request *request) +request_handler_static_finalize (struct request *request) { (void) request; // Nothing to dispose of this far @@ -1828,9 +1828,9 @@ request_handler_static_destroy (struct request *request) struct request_handler g_request_handler_static = { - .try_handle = request_handler_static_try_handle, - .push_cb = request_handler_static_push, - .destroy_cb = request_handler_static_destroy, + .try_handle = request_handler_static_try_handle, + .push_cb = request_handler_static_push, + .finalize_cb = request_handler_static_finalize, }; // --- Client communication handlers ------------------------------------------- @@ -2047,7 +2047,7 @@ client_fcgi_request_push } static void -client_fcgi_request_destroy (struct fcgi_request *req) +client_fcgi_request_finalize (struct fcgi_request *req) { struct client_fcgi_request *request = req->handler_data; request_free (&request->request); @@ -2114,11 +2114,11 @@ client_fcgi_create (EV_P_ int sock_fd) self->client.vtable = &client_fcgi_vtable; fcgi_muxer_init (&self->muxer); - self->muxer.write_cb = client_fcgi_write_cb; - self->muxer.close_cb = client_fcgi_close_cb; - self->muxer.request_start_cb = client_fcgi_request_start; - self->muxer.request_push_cb = client_fcgi_request_push; - self->muxer.request_destroy_cb = client_fcgi_request_destroy; + self->muxer.write_cb = client_fcgi_write_cb; + self->muxer.close_cb = client_fcgi_close_cb; + self->muxer.request_start_cb = client_fcgi_request_start; + self->muxer.request_push_cb = client_fcgi_request_push; + self->muxer.request_finalize_cb = client_fcgi_request_finalize; return &self->client; } From 13892fcd0e2f95b8033b92838996b0b700905652 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Wed, 17 Oct 2018 05:16:17 +0200 Subject: [PATCH 69/79] Clean up client de/allocation --- demo-json-rpc-server.c | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index 8ad1c36..ce50c0f 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -1871,12 +1871,6 @@ struct client_vtable void (*finalize) (struct client *client); }; -static void -client_free (struct client *self) -{ - write_queue_free (&self->write_queue); -} - static void client_write (struct client *self, const void *data, size_t len) { @@ -1902,7 +1896,7 @@ client_destroy (struct client *self) ev_io_stop (EV_DEFAULT_ &self->read_watcher); ev_io_stop (EV_DEFAULT_ &self->write_watcher); xclose (self->socket_fd); - client_free (self); + write_queue_free (&self->write_queue); free (self); try_finish_quit (ctx); @@ -1962,12 +1956,14 @@ close: client_destroy (client); } -static void -client_init (EV_P_ struct client *self, int sock_fd) +/// Create a new instance of a subclass with the given size. +/// The superclass is assumed to be the first member of the structure. +static void * +client_new (EV_P_ size_t size, int sock_fd) { struct server_context *ctx = ev_userdata (loop); + struct client *self = xcalloc (1, size); - memset (self, 0, sizeof *self); self->ctx = ctx; self->write_queue = write_queue_make (); @@ -1984,6 +1980,7 @@ client_init (EV_P_ struct client *self, int sock_fd) LIST_PREPEND (ctx->clients, self); ctx->n_clients++; + return self; } // --- FastCGI client handler -------------------------------------------------- @@ -2109,8 +2106,7 @@ static struct client_vtable client_fcgi_vtable = static struct client * client_fcgi_create (EV_P_ int sock_fd) { - struct client_fcgi *self = xcalloc (1, sizeof *self); - client_init (EV_A_ &self->client, sock_fd); + struct client_fcgi *self = client_new (EV_A_ sizeof *self, sock_fd); self->client.vtable = &client_fcgi_vtable; fcgi_muxer_init (&self->muxer); @@ -2201,8 +2197,7 @@ static struct client_vtable client_scgi_vtable = static struct client * client_scgi_create (EV_P_ int sock_fd) { - struct client_scgi *self = xcalloc (1, sizeof *self); - client_init (EV_A_ &self->client, sock_fd); + struct client_scgi *self = client_new (EV_A_ sizeof *self, sock_fd); self->client.vtable = &client_scgi_vtable; request_init (&self->request); @@ -2298,8 +2293,7 @@ static struct client_vtable client_ws_vtable = static struct client * client_ws_create (EV_P_ int sock_fd) { - struct client_ws *self = xcalloc (1, sizeof *self); - client_init (EV_A_ &self->client, sock_fd); + struct client_ws *self = client_new (EV_A_ sizeof *self, sock_fd); self->client.vtable = &client_ws_vtable; ws_handler_init (&self->handler); From efd500ca3cf4dfdc635ccd71ea8baf5a0d3c5bbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Wed, 17 Oct 2018 06:08:11 +0200 Subject: [PATCH 70/79] Accelerated daemon quitting --- demo-json-rpc-server.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index ce50c0f..cde6ad0 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -2586,10 +2586,14 @@ on_termination_signal (EV_P_ ev_signal *handle, int revents) (void) handle; (void) revents; - // TODO: consider quitting right away if already quitting; - // considering that this may already happen in timeout, it should be OK; - // see on_quit_timeout, just destroy all clients - initiate_quit (ctx); + if (ctx->quitting) + { + // Double C-c from the terminal accelerates the process + LIST_FOR_EACH (struct client, iter, ctx->clients) + client_destroy (iter); + } + else + initiate_quit (ctx); } static void From a3ec0942f8063509a00695d6e702a7f914caa7e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Wed, 17 Oct 2018 08:40:18 +0200 Subject: [PATCH 71/79] Implement basic connection teardown I finally understand the codebase again. It's rather complicated. --- demo-json-rpc-server.c | 238 +++++++++++++++++++++++++---------------- 1 file changed, 146 insertions(+), 92 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index cde6ad0..563d11c 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -47,10 +47,10 @@ enum { PIPE_READ, PIPE_WRITE }; #define FIND_CONTAINER(name, pointer, type, member) \ type *name = CONTAINER_OF (pointer, type, member) -// --- libev helpers ----------------------------------------------------------- +// --- Utilities --------------------------------------------------------------- static bool -flush_queue (struct write_queue *queue, ev_io *watcher) +flush_queue (struct write_queue *queue, int fd) { struct iovec vec[queue->len], *vec_iter = vec; LIST_FOR_EACH (struct write_req, iter, queue->head) @@ -58,24 +58,17 @@ flush_queue (struct write_queue *queue, ev_io *watcher) ssize_t written; again: - written = writev (watcher->fd, vec, N_ELEMENTS (vec)); - if (written < 0) + if ((written = writev (fd, vec, N_ELEMENTS (vec))) >= 0) { - if (errno == EAGAIN) - goto skip; - if (errno == EINTR) - goto again; - return false; + write_queue_processed (queue, written); + return true; } + if (errno == EINTR) + goto again; + if (errno == EAGAIN) + return true; - write_queue_processed (queue, written); - -skip: - if (write_queue_is_empty (queue)) - ev_io_stop (EV_DEFAULT_ watcher); - else - ev_io_start (EV_DEFAULT_ watcher); - return true; + return false; } // --- Logging ----------------------------------------------------------------- @@ -135,16 +128,17 @@ struct fcgi_muxer /// Write data to the underlying transport void (*write_cb) (struct fcgi_muxer *, const void *data, size_t len); - /// Close the underlying transport - // TODO: consider half-close and the subsequent handling + /// Close the underlying transport. You are allowed to destroy the muxer + /// directly from within the callback. void (*close_cb) (struct fcgi_muxer *); /// Start processing a request. Return false if no further action is /// to be done and the request should be finished. - bool (*request_start_cb) (struct fcgi_muxer *, struct fcgi_request *); + bool (*request_start_cb) (struct fcgi_request *); - /// Handle incoming data. "len == 0" means EOF. - void (*request_push_cb) + /// Handle incoming data. "len == 0" means EOF. Returns false if + /// the underlying transport should be closed, this being the last request. + bool (*request_push_cb) (struct fcgi_request *, const void *data, size_t len); /// Destroy the handler's data stored in the request object @@ -230,7 +224,7 @@ fcgi_request_push_params { // TODO: probably check the state of the header parser // TODO: request_start() can return false, end the request here? - (void) self->muxer->request_start_cb (self->muxer, self); + (void) self->muxer->request_start_cb (self); self->state = FCGI_REQUEST_STDIN; } } @@ -282,7 +276,9 @@ fcgi_request_write (struct fcgi_request *self, const void *data, size_t len) } } -static void +/// Mark the request as done. Returns false if the underlying transport +/// should be closed, this being the last request. +static bool fcgi_request_finish (struct fcgi_request *self) { fcgi_request_flush (self); @@ -292,14 +288,26 @@ fcgi_request_finish (struct fcgi_request *self) 0 /* TODO app_status, although ignored */, FCGI_REQUEST_COMPLETE /* TODO protocol_status, may be different */); - if (!(self->flags & FCGI_KEEP_CONN)) - { - // TODO: tear down (shut down) the connection - } + bool should_close = !(self->flags & FCGI_KEEP_CONN); self->muxer->active_requests--; self->muxer->requests[self->request_id] = NULL; fcgi_request_destroy (self); + + // TODO: tear down (shut down) the connection. This is called from: + // + // 1. client_fcgi_request_push <- request_push_cb + // <- fcgi_request_push_stdin <- fcgi_muxer_on_stdin + // <- fcgi_muxer_on_message <- fcgi_parser_push <- fcgi_muxer_push + // <- client_fcgi_push <- client_read_loop + // => in this case no close_cb may be called + // -> need to pass a false boolean aaall the way up, + // then client_fcgi_finalize eventually cleans up the rest + // + // 2. client_fcgi_request_close_cb <- request_finish + // => our direct caller must call fcgi_muxer::close_cb + // -> not very nice to delegate it there + return !should_close; } // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -418,6 +426,7 @@ fcgi_muxer_on_abort_request // TODO: abort the request: let it somehow produce FCGI_END_REQUEST, // make sure to send an stdout EOF record + // TODO: and if that was not a FCGI_KEEP_CONN request, close the transport } static void @@ -1841,23 +1850,24 @@ struct client { LIST_HEADER (struct client) - // XXX: do we really need this here? - struct server_context *ctx; ///< Server context + struct client_vtable *vtable; ///< Client behaviour - int socket_fd; ///< The TCP socket + int socket_fd; ///< The network socket + bool received_eof; ///< Whether EOF has been received yet + bool closing; ///< Whether we're just flushing buffers + bool half_closed; ///< Transport half-closed while closing struct write_queue write_queue; ///< Write queue + ev_timer flush_timeout_watcher; ///< Write queue flush timer ev_io read_watcher; ///< The socket can be read from ev_io write_watcher; ///< The socket can be written to - - struct client_vtable *vtable; ///< Client behaviour }; /// The concrete behaviour to serve a particular client's requests struct client_vtable { /// Process incoming data; "len == 0" means EOF. - /// If the method returns false, the client is destroyed by caller. + /// If the method returns false, client_close() is called by the caller. bool (*push) (struct client *client, const void *data, size_t len); // TODO: optional push_error() to inform about network I/O errors @@ -1885,8 +1895,8 @@ client_write (struct client *self, const void *data, size_t len) static void client_destroy (struct client *self) { - struct server_context *ctx = self->ctx; - + // XXX: this codebase halfway pretends there could be other contexts + struct server_context *ctx = ev_userdata (EV_DEFAULT); LIST_UNLINK (ctx->clients, self); ctx->n_clients--; @@ -1897,63 +1907,111 @@ client_destroy (struct client *self) ev_io_stop (EV_DEFAULT_ &self->write_watcher); xclose (self->socket_fd); write_queue_free (&self->write_queue); + ev_timer_stop (EV_DEFAULT_ &self->flush_timeout_watcher); free (self); try_finish_quit (ctx); } +/// Try to cleanly close the connection, waiting for the remote client to close +/// its own side of the connection as a sign that it has processed all the data +/// it wanted to. The client implementation will not receive any further data. +/// May directly call client_destroy(). +static void +client_close (struct client *self) +{ + if (self->closing) + return; + + self->closing = true; + ev_timer_start (EV_DEFAULT_ &self->flush_timeout_watcher); + ev_feed_event (EV_DEFAULT_ &self->write_watcher, EV_WRITE); + + // We assume the remote client doesn't want our data if it half-closes + if (self->received_eof) + client_destroy (self); +} + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - static bool client_read_loop (EV_P_ struct client *client, ev_io *watcher) { char buf[8192]; - while (true) + ssize_t n_read; +again: + while ((n_read = recv (watcher->fd, buf, sizeof buf, 0)) >= 0) { - ssize_t n_read = recv (watcher->fd, buf, sizeof buf, 0); - if (n_read >= 0) + if (!n_read) { - if (!client->vtable->push (client, buf, n_read)) - return false; - if (!n_read) - break; + // Don't deliver the EOF condition repeatedly + ev_io_stop (EV_A_ watcher); + client->received_eof = true; } - else if (errno == EAGAIN) - return true; - else if (errno != EINTR) + if (!client->closing + && !client->vtable->push (client, buf, n_read)) + { + client_close (client); return false; + } + if (!n_read) + return true; } + if (errno == EINTR) + goto again; + if (errno == EAGAIN) + return true; - // Don't receive the EOF condition repeatedly - ev_io_stop (EV_A_ watcher); - - // We can probably still write, so let's just return - // XXX: if there's nothing to be written, shouldn't we close the connection? - return true; + client_destroy (client); + return false; } static void -on_client_ready (EV_P_ ev_io *watcher, int revents) +on_client_readable (EV_P_ ev_io *watcher, int revents) { struct client *client = watcher->data; - // XXX: although read and write are in a sequence, if we create response - // data, we'll still likely need to go back to the event loop. + (void) revents; - if (revents & EV_READ) - if (!client_read_loop (EV_A_ client, watcher)) - goto close; - if (revents & EV_WRITE) - // TODO: add "closing link" functionality -> automatic shutdown - // (half-close) once we manage to flush the write buffer, - // which is logically followed by waiting for an EOF from the client - // TODO: some sort of "on_buffers_flushed" callback for streaming huge - // chunks of external (or generated) data. - if (!flush_queue (&client->write_queue, watcher)) - goto close; - return; + if (client_read_loop (EV_A_ client, watcher) + && client->closing && client->received_eof) + client_destroy (client); +} -close: - client_destroy (client); +static void +on_client_writable (EV_P_ ev_io *watcher, int revents) +{ + struct client *client = watcher->data; + (void) loop; + (void) revents; + + // TODO: some sort of "on_buffers_flushed" callback for streaming huge + // chunks of external (or generated) data. That will need to be + // forwarded to "struct request_handler". + if (!flush_queue (&client->write_queue, watcher->fd)) + { + client_destroy (client); + return; + } + if (!write_queue_is_empty (&client->write_queue)) + return; + + ev_io_stop (EV_A_ watcher); + if (client->closing && !client->half_closed) + { + if (!shutdown (client->socket_fd, SHUT_WR)) + client->half_closed = true; + else + client_destroy (client); + } +} + +static void +on_client_timeout (EV_P_ ev_timer *watcher, int revents) +{ + (void) loop; + (void) revents; + + client_destroy (watcher->data); } /// Create a new instance of a subclass with the given size. @@ -1964,14 +2022,15 @@ client_new (EV_P_ size_t size, int sock_fd) struct server_context *ctx = ev_userdata (loop); struct client *self = xcalloc (1, size); - self->ctx = ctx; self->write_queue = write_queue_make (); + ev_timer_init (&self->flush_timeout_watcher, on_client_timeout, 5., 0.); + self->flush_timeout_watcher.data = self; set_blocking (sock_fd, false); self->socket_fd = sock_fd; - ev_io_init (&self->read_watcher, on_client_ready, sock_fd, EV_READ); - ev_io_init (&self->write_watcher, on_client_ready, sock_fd, EV_WRITE); + ev_io_init (&self->read_watcher, on_client_readable, sock_fd, EV_READ); + ev_io_init (&self->write_watcher, on_client_writable, sock_fd, EV_WRITE); self->read_watcher.data = self; self->write_watcher.data = self; @@ -2010,37 +2069,36 @@ static void client_fcgi_request_close_cb (struct request *req) { FIND_CONTAINER (self, req, struct client_fcgi_request, request); - // No more data to send, terminate the substream/request - // XXX: this will most probably end up with client_fcgi_request_destroy(), - // we might or might not need to defer this action - fcgi_request_finish (self->fcgi_request); + struct fcgi_muxer *muxer = self->fcgi_request->muxer; + // No more data to send, terminate the substream/request, + // and also the transport if the client didn't specifically ask to keep it + if (!fcgi_request_finish (self->fcgi_request)) + muxer->close_cb (muxer); } // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - static bool -client_fcgi_request_start - (struct fcgi_muxer *mux, struct fcgi_request *fcgi_request) +client_fcgi_request_start (struct fcgi_request *fcgi_request) { - FIND_CONTAINER (self, mux, struct client_fcgi, muxer); - struct client_fcgi_request *request = fcgi_request->handler_data = xcalloc (1, sizeof *request); request->fcgi_request = fcgi_request; request_init (&request->request); - request->request.ctx = self->client.ctx; + request->request.ctx = ev_userdata (EV_DEFAULT); request->request.write_cb = client_fcgi_request_write_cb; request->request.close_cb = client_fcgi_request_close_cb; return request_start (&request->request, &fcgi_request->headers); } -static void +static bool client_fcgi_request_push (struct fcgi_request *req, const void *data, size_t len) { struct client_fcgi_request *request = req->handler_data; - request_push (&request->request, data, len); + return request_push (&request->request, data, len) + || fcgi_request_finish (req); } static void @@ -2064,9 +2122,7 @@ static void client_fcgi_close_cb (struct fcgi_muxer *mux) { FIND_CONTAINER (self, mux, struct client_fcgi, muxer); - // FIXME: we should probably call something like client_shutdown(), - // which may have an argument whether we should really use close() - client_destroy (&self->client); + client_close (&self->client); } // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -2137,11 +2193,9 @@ client_scgi_write_cb (struct request *req, const void *data, size_t len) static void client_scgi_close_cb (struct request *req) { - // NOTE: this rather really means "close me [the request]" FIND_CONTAINER (self, req, struct client_scgi, request); - // FIXME: we should probably call something like client_shutdown(), - // which may have an argument whether we should really use close() - client_destroy (&self->client); + // NOTE: this rather really means "close me [the request]" + client_close (&self->client); } static bool @@ -2201,7 +2255,6 @@ client_scgi_create (EV_P_ int sock_fd) self->client.vtable = &client_scgi_vtable; request_init (&self->request); - self->request.ctx = self->client.ctx; self->request.write_cb = client_scgi_write_cb; self->request.close_cb = client_scgi_close_cb; @@ -2231,8 +2284,9 @@ client_ws_on_message (struct ws_handler *handler, return false; } + struct server_context *ctx = ev_userdata (EV_DEFAULT); struct str response = str_make (); - process_json_rpc (self->client.ctx, data, len, &response); + process_json_rpc (ctx, data, len, &response); if (response.len) ws_handler_send (&self->handler, WS_OPCODE_TEXT, response.str, response.len); From cf56921c4e1959bb3962b4f99ee4a1e7665c57d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Wed, 17 Oct 2018 23:26:11 +0200 Subject: [PATCH 72/79] Allow WebSockets to micromanage shutdowns They have their reasons, mostly event-related. --- demo-json-rpc-server.c | 59 +++++++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index 563d11c..2b99419 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -1854,10 +1854,11 @@ struct client int socket_fd; ///< The network socket bool received_eof; ///< Whether EOF has been received yet - bool closing; ///< Whether we're just flushing buffers - bool half_closed; ///< Transport half-closed while closing + bool flushing; ///< No more data to write, send FIN + bool closing; ///< No more data to read or write + bool half_closed; ///< Conn. half-closed while flushing struct write_queue write_queue; ///< Write queue - ev_timer flush_timeout_watcher; ///< Write queue flush timer + ev_timer close_timeout_watcher; ///< Write queue flush timer ev_io read_watcher; ///< The socket can be read from ev_io write_watcher; ///< The socket can be written to @@ -1881,17 +1882,6 @@ struct client_vtable void (*finalize) (struct client *client); }; -static void -client_write (struct client *self, const void *data, size_t len) -{ - struct write_req *req = xcalloc (1, sizeof *req); - req->data.iov_base = memcpy (xmalloc (len), data, len); - req->data.iov_len = len; - - write_queue_add (&self->write_queue, req); - ev_io_start (EV_DEFAULT_ &self->write_watcher); -} - static void client_destroy (struct client *self) { @@ -1907,12 +1897,36 @@ client_destroy (struct client *self) ev_io_stop (EV_DEFAULT_ &self->write_watcher); xclose (self->socket_fd); write_queue_free (&self->write_queue); - ev_timer_stop (EV_DEFAULT_ &self->flush_timeout_watcher); + ev_timer_stop (EV_DEFAULT_ &self->close_timeout_watcher); free (self); try_finish_quit (ctx); } +static void +client_write (struct client *self, const void *data, size_t len) +{ + if (!soft_assert (!self->flushing)) + return; + + struct write_req *req = xcalloc (1, sizeof *req); + req->data.iov_base = memcpy (xmalloc (len), data, len); + req->data.iov_len = len; + + write_queue_add (&self->write_queue, req); + ev_io_start (EV_DEFAULT_ &self->write_watcher); +} + +/// Half-close the connection from our side once the write_queue is flushed. +/// It is the caller's responsibility to destroy the connection upon EOF. +// XXX: or we might change on_client_readable to do it anyway, seems safe +static void +client_shutdown (struct client *self) +{ + self->flushing = true; + ev_feed_event (EV_DEFAULT_ &self->write_watcher, EV_WRITE); +} + /// Try to cleanly close the connection, waiting for the remote client to close /// its own side of the connection as a sign that it has processed all the data /// it wanted to. The client implementation will not receive any further data. @@ -1924,8 +1938,8 @@ client_close (struct client *self) return; self->closing = true; - ev_timer_start (EV_DEFAULT_ &self->flush_timeout_watcher); - ev_feed_event (EV_DEFAULT_ &self->write_watcher, EV_WRITE); + ev_timer_start (EV_DEFAULT_ &self->close_timeout_watcher); + client_shutdown (self); // We assume the remote client doesn't want our data if it half-closes if (self->received_eof) @@ -1996,7 +2010,7 @@ on_client_writable (EV_P_ ev_io *watcher, int revents) return; ev_io_stop (EV_A_ watcher); - if (client->closing && !client->half_closed) + if (client->flushing && !client->half_closed) { if (!shutdown (client->socket_fd, SHUT_WR)) client->half_closed = true; @@ -2023,8 +2037,8 @@ client_new (EV_P_ size_t size, int sock_fd) struct client *self = xcalloc (1, size); self->write_queue = write_queue_make (); - ev_timer_init (&self->flush_timeout_watcher, on_client_timeout, 5., 0.); - self->flush_timeout_watcher.data = self; + ev_timer_init (&self->close_timeout_watcher, on_client_timeout, 5., 0.); + self->close_timeout_watcher.data = self; set_blocking (sock_fd, false); self->socket_fd = sock_fd; @@ -2305,10 +2319,7 @@ static void client_ws_close_cb (struct ws_handler *handler, bool half_close) { FIND_CONTAINER (self, handler, struct client_ws, handler); - if (half_close) - ; // FIXME: we should probably call something like client_shutdown() - else - client_destroy (&self->client); + (half_close ? client_shutdown : client_destroy) (&self->client); } // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - From 253e35e1e43b8c7f8ea8de7b94556a4ef6cf0d95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Wed, 17 Oct 2018 23:34:59 +0200 Subject: [PATCH 73/79] Wrap request::write_cb in a function --- demo-json-rpc-server.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index 2b99419..9552146 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -1563,6 +1563,13 @@ request_free (struct request *self) self->handler->finalize_cb (self); } +/// Write request CGI response data, intended for use by request handlers +static void +request_write (struct request *self, const void *data, size_t len) +{ + self->write_cb (self, data, len); +} + /// This function is only intended to be run from asynchronous event handlers /// such as timers, not as a direct result of starting the request or receiving /// request data. CALLING THIS MAY CAUSE THE REQUEST TO BE DESTROYED. @@ -1599,7 +1606,7 @@ request_start (struct request *self, struct str_map *headers) struct str response = str_make (); str_append (&response, "Status: 404 Not Found\n"); str_append (&response, "Content-Type: text/plain\n\n"); - self->write_cb (self, response.str, response.len); + request_write (self, response.str, response.len); str_free (&response); return false; } @@ -1654,7 +1661,7 @@ request_handler_json_rpc_push str_append (&response, "Status: 200 OK\n"); str_append_printf (&response, "Content-Type: %s\n\n", "application/json"); process_json_rpc (request->ctx, buf->str, buf->len, &response); - request->write_cb (request, response.str, response.len); + request_write (request, response.str, response.len); str_free (&response); return false; } @@ -1770,7 +1777,7 @@ request_handler_static_try_handle str_append (&response, "Content-Type: text/plain\n\n"); str_append_printf (&response, "File %s was not found on this server\n", suffix); - request->write_cb (request, response.str, response.len); + request_write (request, response.str, response.len); str_free (&response); free (suffix); @@ -1794,17 +1801,17 @@ request_handler_static_try_handle struct str response = str_make (); str_append (&response, "Status: 200 OK\n"); str_append_printf (&response, "Content-Type: %s\n\n", mime_type); - request->write_cb (request, response.str, response.len); + request_write (request, response.str, response.len); str_free (&response); free (mime_type); // Write the chunk we've used to help us with magic detection; // obviously we have to do it after we've written the headers if (len) - request->write_cb (request, buf, len); + request_write (request, buf, len); while ((len = fread (buf, 1, sizeof buf, fp))) - request->write_cb (request, buf, len); + request_write (request, buf, len); fclose (fp); // TODO: this should rather not be returned all at once but in chunks; From 580f0a0c5960d830f49c4cddbfdf869c1d262a0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Wed, 17 Oct 2018 23:56:11 +0200 Subject: [PATCH 74/79] Synthesize EOF events in SCGI --- demo-json-rpc-server.c | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index 9552146..1ae1bad 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -1544,6 +1544,8 @@ struct request_handler /// Handle incoming data. "len == 0" means EOF. /// Returns false if there is no more processing to be done. + // FIXME: the EOF may or may not be delivered when request is cut short, + // we should fix FastCGI not to deliver it on CONTENT_LENGTH mismatch bool (*push_cb) (struct request *request, const void *data, size_t len); /// Destroy the handler's data stored in the request object @@ -1655,7 +1657,8 @@ request_handler_json_rpc_push } // TODO: check buf.len against CONTENT_LENGTH; if it's less, then the - // client hasn't been successful in transferring all of its data + // client hasn't been successful in transferring all of its data. + // See also comment on request_handler::push_cb. struct str response = str_make (); str_append (&response, "Status: 200 OK\n"); @@ -2202,6 +2205,7 @@ struct client_scgi struct client client; ///< Parent class struct scgi_parser parser; ///< SCGI stream parser struct request request; ///< Request (only one per connection) + unsigned long remaining_content; ///< Length of input data to be seen }; static void @@ -2223,6 +2227,12 @@ static bool client_scgi_on_headers_read (void *user_data) { struct client_scgi *self = user_data; + const char *cl = str_map_find (&self->parser.headers, "CONTENT_LENGTH"); + if (!cl || !xstrtoul (&self->remaining_content, cl, 10)) + { + print_debug ("SCGI request with invalid or missing CONTENT_LENGTH"); + return false; + } return request_start (&self->request, &self->parser.headers); } @@ -2230,11 +2240,19 @@ static bool client_scgi_on_content (void *user_data, const void *data, size_t len) { struct client_scgi *self = user_data; + if (len > self->remaining_content) + { + print_debug ("SCGI request got more data than CONTENT_LENGTH"); + return false; + } + // We're in a slight disagreement with the specification since + // this tries to write output before it has read all the input + if (!request_push (&self->request, data, len)) + return false; - // XXX: do we have to count CONTENT_LENGTH and supply our own EOF? - // If we do produce our own EOF, we should probably make sure we don't - // send it twice in a row. - return request_push (&self->request, data, len); + // Signalise end of input to the request handler + return (self->remaining_content -= len) != 0 + || request_push (&self->request, NULL, 0); } // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - From 62945cceb3b2f9fae0fd39842448ebfeba378fa9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Thu, 18 Oct 2018 02:55:55 +0200 Subject: [PATCH 75/79] Finish the WebSocket backend Of course, everything so far hasn't been tested much. --- demo-json-rpc-server.c | 275 ++++++++++++++++++++++++----------------- 1 file changed, 164 insertions(+), 111 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index 1ae1bad..60a7173 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -540,9 +540,9 @@ enum ws_handler_state { WS_HANDLER_CONNECTING, ///< Parsing HTTP WS_HANDLER_OPEN, ///< Parsing WebSockets frames - WS_HANDLER_CLOSING, ///< Closing the connection - WS_HANDLER_ALMOST_DEAD, ///< Closing connection after failure - WS_HANDLER_CLOSED ///< Dead + WS_HANDLER_CLOSING, ///< Partial closure by us + WS_HANDLER_FLUSHING, ///< Just waiting for client EOF + WS_HANDLER_CLOSED ///< Dead, both sides closed }; struct ws_handler @@ -584,6 +584,7 @@ struct ws_handler // TODO: void (*on_handshake) (protocols) that will allow the user // to choose any sub-protocol, if the client has provided any. // This may render "on_connected" unnecessary. + // Should also enable failing the handshake. /// Called after successfuly connecting (handshake complete) bool (*on_connected) (struct ws_handler *); @@ -626,36 +627,41 @@ static void ws_handler_close (struct ws_handler *self, enum ws_status close_code, const char *reason, size_t len) { + hard_assert (self->state == WS_HANDLER_OPEN); + struct str payload = str_make (); str_pack_u16 (&payload, close_code); // XXX: maybe accept a null-terminated string on input? Has to be UTF-8 a/w str_append_data (&payload, reason, len); ws_handler_send_control (self, WS_OPCODE_CLOSE, payload.str, payload.len); - - // Close initiated by us; the reason is null-terminated within `payload' - if (self->on_close) - self->on_close (self, close_code, payload.str + 2); + self->close_cb (self, true /* half_close */); self->state = WS_HANDLER_CLOSING; str_free (&payload); } -static void -ws_handler_fail (struct ws_handler *self, enum ws_status close_code) +static bool +ws_handler_fail_connection (struct ws_handler *self, enum ws_status close_code) { - ws_handler_close (self, close_code, NULL, 0); - self->state = WS_HANDLER_ALMOST_DEAD; + hard_assert (self->state == WS_HANDLER_OPEN + || self->state == WS_HANDLER_CLOSING); - // TODO: set the close timer, ignore all further incoming input (either set - // some flag for the case that we're in the middle of ws_handler_push(), - // and/or add a mechanism to stop the caller from polling the socket for - // reads). - // TODO: make sure we don't send pings after the close + if (self->state == WS_HANDLER_OPEN) + ws_handler_close (self, close_code, NULL, 0); + + self->state = WS_HANDLER_FLUSHING; + if (self->on_close) + self->on_close (self, WS_STATUS_ABNORMAL_CLOSURE, ""); + + ev_timer_stop (EV_DEFAULT_ &self->ping_timer); + ev_timer_set (&self->close_timeout_watcher, self->close_timeout, 0.); + ev_timer_start (EV_DEFAULT_ &self->close_timeout_watcher); + return false; } // TODO: add support for fragmented responses static void -ws_handler_send (struct ws_handler *self, +ws_handler_send_frame (struct ws_handler *self, enum ws_opcode opcode, const void *data, size_t len) { if (!soft_assert (self->state == WS_HANDLER_OPEN)) @@ -697,24 +703,26 @@ ws_handler_on_frame_header (void *user_data, const struct ws_parser *parser) || (!ws_is_control_frame (parser->opcode) && (self->expecting_continuation && parser->opcode != WS_OPCODE_CONT)) || parser->payload_len >= 0x8000000000000000ULL) - ws_handler_fail (self, WS_STATUS_PROTOCOL_ERROR); - else if (parser->payload_len > self->max_payload_len) - ws_handler_fail (self, WS_STATUS_MESSAGE_TOO_BIG); - else - return true; - return false; + return ws_handler_fail_connection (self, WS_STATUS_PROTOCOL_ERROR); + + if (parser->payload_len > self->max_payload_len + || (self->expecting_continuation && + self->message_data.len + parser->payload_len > self->max_payload_len)) + return ws_handler_fail_connection (self, WS_STATUS_MESSAGE_TOO_BIG); + return true; } static bool -ws_handler_on_protocol_close +ws_handler_on_control_close (struct ws_handler *self, const struct ws_parser *parser) { + hard_assert (self->state == WS_HANDLER_OPEN + || self->state == WS_HANDLER_CLOSING); struct msg_unpacker unpacker = msg_unpacker_make (parser->input.str, parser->payload_len); char *reason = NULL; uint16_t close_code = WS_STATUS_NO_STATUS_RECEIVED; - if (parser->payload_len >= 2) { (void) msg_unpacker_u16 (&unpacker, &close_code); @@ -723,17 +731,29 @@ ws_handler_on_protocol_close else reason = xstrdup (""); - if (self->state != WS_HANDLER_CLOSING) + if (close_code < 1000 || close_code > 4999) + // XXX: invalid close code: maybe we should fail the connection instead + close_code = WS_STATUS_PROTOCOL_ERROR; + + if (self->state == WS_HANDLER_OPEN) { // Close initiated by the client + // FIXME: not sending the potentially different close_code ws_handler_send_control (self, WS_OPCODE_CLOSE, parser->input.str, parser->payload_len); + + self->state = WS_HANDLER_FLUSHING; if (self->on_close) self->on_close (self, close_code, reason); } + else + self->state = WS_HANDLER_FLUSHING; free (reason); - self->state = WS_HANDLER_ALMOST_DEAD; + + ev_timer_stop (EV_DEFAULT_ &self->ping_timer); + ev_timer_set (&self->close_timeout_watcher, self->close_timeout, 0.); + ev_timer_start (EV_DEFAULT_ &self->close_timeout_watcher); return true; } @@ -744,21 +764,18 @@ ws_handler_on_control_frame switch (parser->opcode) { case WS_OPCODE_CLOSE: - return ws_handler_on_protocol_close (self, parser); + return ws_handler_on_control_close (self, parser); case WS_OPCODE_PING: ws_handler_send_control (self, WS_OPCODE_PONG, parser->input.str, parser->payload_len); break; case WS_OPCODE_PONG: - // XXX: maybe we should check the payload + // TODO: check the payload self->received_pong = true; break; default: // Unknown control frame - ws_handler_fail (self, WS_STATUS_PROTOCOL_ERROR); - // FIXME: we shouldn't close the connection right away; - // also check other places - return false; + return ws_handler_fail_connection (self, WS_STATUS_PROTOCOL_ERROR); } return true; } @@ -769,29 +786,19 @@ ws_handler_on_frame (void *user_data, const struct ws_parser *parser) struct ws_handler *self = user_data; if (ws_is_control_frame (parser->opcode)) return ws_handler_on_control_frame (self, parser); - - // TODO: do this rather in "on_frame_header" - if (self->message_data.len + parser->payload_len > self->max_payload_len) - { - ws_handler_fail (self, WS_STATUS_MESSAGE_TOO_BIG); - return false; - } - if (!self->expecting_continuation) self->message_opcode = parser->opcode; str_append_data (&self->message_data, parser->input.str, parser->payload_len); - self->expecting_continuation = !parser->is_fin; - - if (!parser->is_fin) + if ((self->expecting_continuation = !parser->is_fin)) return true; if (self->message_opcode == WS_OPCODE_TEXT && !utf8_validate (self->message_data.str, self->message_data.len)) { - ws_handler_fail (self, WS_STATUS_INVALID_PAYLOAD_DATA); - return false; + return ws_handler_fail_connection + (self, WS_STATUS_INVALID_PAYLOAD_DATA); } bool result = true; @@ -799,6 +806,8 @@ ws_handler_on_frame (void *user_data, const struct ws_parser *parser) result = self->on_message (self, self->message_opcode, self->message_data.str, self->message_data.len); str_reset (&self->message_data); + // TODO: if (!result), either replace this with a state check, + // or make sure to change the state return result; } @@ -810,11 +819,10 @@ ws_handler_on_ping_timer (EV_P_ ev_timer *watcher, int revents) struct ws_handler *self = watcher->data; if (!self->received_pong) - { - // TODO: close/fail the connection? - } + ws_handler_fail_connection (self, 4000); else { + // TODO: be an annoying server and send a nonce in the data ws_handler_send_control (self, WS_OPCODE_PING, NULL, 0); ev_timer_again (EV_A_ watcher); } @@ -823,20 +831,38 @@ ws_handler_on_ping_timer (EV_P_ ev_timer *watcher, int revents) static void ws_handler_on_close_timeout (EV_P_ ev_timer *watcher, int revents) { + (void) loop; (void) revents; struct ws_handler *self = watcher->data; - // TODO: anything else to do here? Invalidate our state? - if (self->close_cb) - self->close_cb (self, false /* half_close */); + hard_assert (self->state == WS_HANDLER_OPEN + || self->state == WS_HANDLER_CLOSING); + + if (self->state == WS_HANDLER_CLOSING + && self->on_close) + self->on_close (self, WS_STATUS_ABNORMAL_CLOSURE, "close timeout"); + + self->state = WS_HANDLER_CLOSED; + self->close_cb (self, false /* half_close */); } static void ws_handler_on_handshake_timeout (EV_P_ ev_timer *watcher, int revents) { + (void) loop; (void) revents; struct ws_handler *self = watcher->data; - // TODO + + // XXX: this is a no-op, since this currently doesn't even call shutdown + // immediately but postpones it until later + self->close_cb (self, true /* half_close */); + self->state = WS_HANDLER_FLUSHING; + + if (self->on_close) + self->on_close (self, WS_STATUS_ABNORMAL_CLOSURE, "handshake timeout"); + + self->state = WS_HANDLER_CLOSED; + self->close_cb (self, false /* half_close */); } static void @@ -991,6 +1017,7 @@ ws_handler_on_url (http_parser *parser, const char *at, size_t len) #define HTTP_400_BAD_REQUEST "400 Bad Request" #define HTTP_405_METHOD_NOT_ALLOWED "405 Method Not Allowed" #define HTTP_417_EXPECTATION_FAILED "407 Expectation Failed" +#define HTTP_426_UPGRADE_REQUIRED "426 Upgrade Required" #define HTTP_505_VERSION_NOT_SUPPORTED "505 HTTP Version Not Supported" static void @@ -1024,43 +1051,47 @@ ws_handler_http_responsev (struct ws_handler *self, str_free (&response); } -static void -ws_handler_http_response (struct ws_handler *self, const char *status, ...) +static bool +ws_handler_fail_handshake (struct ws_handler *self, const char *status, ...) { - struct strv v = strv_make (); - va_list ap; va_start (ap, status); const char *s; + struct strv v = strv_make (); while ((s = va_arg (ap, const char *))) strv_append (&v, s); va_end (ap); - ws_handler_http_responsev (self, status, v.vector); strv_free (&v); + + self->close_cb (self, true /* half_close */); + self->state = WS_HANDLER_FLUSHING; + + if (self->on_close) + self->on_close (self, WS_STATUS_ABNORMAL_CLOSURE, status); + return false; } -#define FAIL_HANDSHAKE(status, ...) \ - BLOCK_START \ - self->state = WS_HANDLER_ALMOST_DEAD; \ - ws_handler_http_response (self, (status), __VA_ARGS__); \ - return false; \ - BLOCK_END +#define FAIL_HANDSHAKE(...) \ + return ws_handler_fail_handshake (self, __VA_ARGS__, NULL) static bool ws_handler_finish_handshake (struct ws_handler *self) { - // XXX: we probably shouldn't use 505 to reject the minor version but w/e - if (self->hp.http_major != 1 || self->hp.http_minor < 1) - FAIL_HANDSHAKE (HTTP_505_VERSION_NOT_SUPPORTED, NULL); if (self->hp.method != HTTP_GET) - FAIL_HANDSHAKE (HTTP_405_METHOD_NOT_ALLOWED, "Allow: GET", NULL); + FAIL_HANDSHAKE (HTTP_405_METHOD_NOT_ALLOWED, "Allow: GET"); + + // Technically, it must be /at least/ 1.1 but no other 1.x version of HTTP + // is going to happen and 2.x is entirely incompatible + // XXX: we probably shouldn't use 505 to reject the minor version but w/e + if (self->hp.http_major != 1 || self->hp.http_minor != 1) + FAIL_HANDSHAKE (HTTP_505_VERSION_NOT_SUPPORTED); // Your expectations are way too high if (str_map_find (&self->headers, "Expect")) - FAIL_HANDSHAKE (HTTP_417_EXPECTATION_FAILED, NULL); + FAIL_HANDSHAKE (HTTP_417_EXPECTATION_FAILED); // Reject URLs specifying the schema and host; we're not parsing that // TODO: actually do parse this and let our user decide if it matches @@ -1068,11 +1099,11 @@ ws_handler_finish_handshake (struct ws_handler *self) if (http_parser_parse_url (self->url.str, self->url.len, false, &url) || (url.field_set & (1 << UF_SCHEMA | 1 << UF_HOST | 1 << UF_PORT)) || !str_map_find (&self->headers, "Host")) - FAIL_HANDSHAKE (HTTP_400_BAD_REQUEST, NULL); + FAIL_HANDSHAKE (HTTP_400_BAD_REQUEST); const char *connection = str_map_find (&self->headers, "Connection"); if (!connection || strcasecmp_ascii (connection, "Upgrade")) - FAIL_HANDSHAKE (HTTP_400_BAD_REQUEST, NULL); + FAIL_HANDSHAKE (HTTP_400_BAD_REQUEST); // Check if we can actually upgrade the protocol to WebSockets const char *upgrade = str_map_find (&self->headers, "Upgrade"); @@ -1088,7 +1119,8 @@ ws_handler_finish_handshake (struct ws_handler *self) http_protocol_destroy (iter); } if (!can_upgrade) - FAIL_HANDSHAKE (HTTP_400_BAD_REQUEST, NULL); + FAIL_HANDSHAKE (HTTP_426_UPGRADE_REQUIRED, + "Upgrade: websocket", SEC_WS_VERSION ": 13"); // Okay, we're finally past the basic HTTP/1.1 stuff const char *key = str_map_find (&self->headers, SEC_WS_KEY); @@ -1098,19 +1130,17 @@ ws_handler_finish_handshake (struct ws_handler *self) const char *extensions = str_map_find (&self->headers, SEC_WS_EXTENSIONS); */ - if (!key) - FAIL_HANDSHAKE (HTTP_400_BAD_REQUEST, NULL); + if (!version) + FAIL_HANDSHAKE (HTTP_400_BAD_REQUEST); + if (strcmp (version, "13")) + FAIL_HANDSHAKE (HTTP_400_BAD_REQUEST, SEC_WS_VERSION ": 13"); struct str tmp = str_make (); - bool key_is_valid = base64_decode (key, false, &tmp) && tmp.len == 16; + bool key_is_valid = key + && base64_decode (key, false, &tmp) && tmp.len == 16; str_free (&tmp); if (!key_is_valid) - FAIL_HANDSHAKE (HTTP_400_BAD_REQUEST, NULL); - - if (!version) - FAIL_HANDSHAKE (HTTP_400_BAD_REQUEST, NULL); - if (strcmp (version, "13")) - FAIL_HANDSHAKE (HTTP_400_BAD_REQUEST, SEC_WS_VERSION ": 13", NULL); + FAIL_HANDSHAKE (HTTP_400_BAD_REQUEST); struct strv fields = strv_make (); strv_append_args (&fields, @@ -1130,6 +1160,7 @@ ws_handler_finish_handshake (struct ws_handler *self) strv_free (&fields); + self->state = WS_HANDLER_OPEN; ev_timer_init (&self->ping_timer, ws_handler_on_ping_timer, self->ping_interval, 0); ev_timer_start (EV_DEFAULT_ &self->ping_timer); @@ -1141,40 +1172,62 @@ ws_handler_finish_handshake (struct ws_handler *self) static void ws_handler_start (struct ws_handler *self) { + hard_assert (self->state == WS_HANDLER_CONNECTING); + ev_timer_set (&self->handshake_timeout_watcher, self->handshake_timeout, 0.); ev_timer_start (EV_DEFAULT_ &self->handshake_timeout_watcher); } +// The client should normally never close the connection, assume that it's +// either received an EOF from our side, or that it doesn't care about our data +// anymore, having called close() already +static bool +ws_handler_push_eof (struct ws_handler *self) +{ + switch (self->state) + { + case WS_HANDLER_CONNECTING: + ev_timer_stop (EV_DEFAULT_ &self->handshake_timeout_watcher); + + self->state = WS_HANDLER_FLUSHING; + if (self->on_close) + self->on_close (self, WS_STATUS_ABNORMAL_CLOSURE, "unexpected EOF"); + break; + case WS_HANDLER_OPEN: + ev_timer_stop (EV_DEFAULT_ &self->ping_timer); + // Fall-through + case WS_HANDLER_CLOSING: + self->state = WS_HANDLER_CLOSED; + if (self->on_close) + self->on_close (self, WS_STATUS_ABNORMAL_CLOSURE, ""); + // Fall-through + case WS_HANDLER_FLUSHING: + ev_timer_stop (EV_DEFAULT_ &self->close_timeout_watcher); + break; + default: + soft_assert(self->state != WS_HANDLER_CLOSED); + } + self->state = WS_HANDLER_CLOSED; + return false; +} + /// Push data to the WebSocket handler. "len == 0" means EOF. +/// You are expected to close the connection and dispose of the handler +/// when the function returns false. static bool ws_handler_push (struct ws_handler *self, const void *data, size_t len) { - // TODO: make sure all timers are stopped appropriately - if (!len) - { - ev_timer_stop (EV_DEFAULT_ &self->handshake_timeout_watcher); + return ws_handler_push_eof (self); - if (self->state == WS_HANDLER_OPEN) - { - if (self->on_close) - self->on_close (self, WS_STATUS_ABNORMAL_CLOSURE, ""); - } - else - { - // TODO: anything to do besides just closing the connection? - } - - self->state = WS_HANDLER_CLOSED; - return false; - } - - if (self->state == WS_HANDLER_ALMOST_DEAD) + if (self->state == WS_HANDLER_FLUSHING) // We're waiting for an EOF from the client, must not process data return true; + if (self->state != WS_HANDLER_CONNECTING) - return ws_parser_push (&self->parser, data, len); + return soft_assert (self->state != WS_HANDLER_CLOSED) + && ws_parser_push (&self->parser, data, len); // The handshake hasn't been done yet, process HTTP headers static const http_parser_settings http_settings = @@ -1185,8 +1238,8 @@ ws_handler_push (struct ws_handler *self, const void *data, size_t len) .on_url = ws_handler_on_url, }; - size_t n_parsed = http_parser_execute (&self->hp, - &http_settings, data, len); + size_t n_parsed = + http_parser_execute (&self->hp, &http_settings, data, len); if (self->hp.upgrade) { @@ -1195,12 +1248,10 @@ ws_handler_push (struct ws_handler *self, const void *data, size_t len) // The handshake hasn't been finished, yet there is more data // to be processed after the headers already if (len - n_parsed) - FAIL_HANDSHAKE (HTTP_400_BAD_REQUEST, NULL); + FAIL_HANDSHAKE (HTTP_400_BAD_REQUEST); if (!ws_handler_finish_handshake (self)) return false; - - self->state = WS_HANDLER_OPEN; if (self->on_connected) return self->on_connected (self); return true; @@ -1217,7 +1268,7 @@ ws_handler_push (struct ws_handler *self, const void *data, size_t len) print_debug ("WS handshake failed: %s", http_errno_description (err)); - FAIL_HANDSHAKE (HTTP_400_BAD_REQUEST, NULL); + FAIL_HANDSHAKE (HTTP_400_BAD_REQUEST); } return true; } @@ -2319,15 +2370,15 @@ client_ws_on_message (struct ws_handler *handler, FIND_CONTAINER (self, handler, struct client_ws, handler); if (type != WS_OPCODE_TEXT) { - ws_handler_fail (&self->handler, WS_STATUS_UNSUPPORTED_DATA); - return false; + return ws_handler_fail_connection + (&self->handler, WS_STATUS_UNSUPPORTED_DATA); } struct server_context *ctx = ev_userdata (EV_DEFAULT); struct str response = str_make (); process_json_rpc (ctx, data, len, &response); if (response.len) - ws_handler_send (&self->handler, + ws_handler_send_frame (&self->handler, WS_OPCODE_TEXT, response.str, response.len); str_free (&response); return true; @@ -2353,6 +2404,7 @@ static bool client_ws_push (struct client *client, const void *data, size_t len) { FIND_CONTAINER (self, client, struct client_ws, client); + // client_close() will correctly destroy the client on EOF return ws_handler_push (&self->handler, data, len); } @@ -2361,7 +2413,8 @@ client_ws_shutdown (struct client *client) { FIND_CONTAINER (self, client, struct client_ws, client); if (self->handler.state == WS_HANDLER_CONNECTING) - ; // TODO: abort the connection immediately + // No on_close, no problem + client_destroy (&self->client); else if (self->handler.state == WS_HANDLER_OPEN) ws_handler_close (&self->handler, WS_STATUS_GOING_AWAY, NULL, 0); } From d883f4cc71a1f9db5cc4cec2bc60df6cba1f8833 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Thu, 18 Oct 2018 04:13:59 +0200 Subject: [PATCH 76/79] Finish the FastCGI backend Bump liberty, also fixing SCGI. --- demo-json-rpc-server.c | 154 ++++++++++++++++++++--------------------- liberty | 2 +- 2 files changed, 75 insertions(+), 81 deletions(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index 60a7173..50e203f 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -194,6 +194,8 @@ fcgi_request_new (void) self->hdr_parser = fcgi_nv_parser_make (); self->hdr_parser.output = &self->headers; + + self->output_buffer = str_make (); return self; } @@ -208,40 +210,6 @@ fcgi_request_destroy (struct fcgi_request *self) free (self); } -static void -fcgi_request_push_params - (struct fcgi_request *self, const void *data, size_t len) -{ - if (self->state != FCGI_REQUEST_PARAMS) - { - // TODO: probably reject the request - return; - } - - if (len) - fcgi_nv_parser_push (&self->hdr_parser, data, len); - else - { - // TODO: probably check the state of the header parser - // TODO: request_start() can return false, end the request here? - (void) self->muxer->request_start_cb (self); - self->state = FCGI_REQUEST_STDIN; - } -} - -static void -fcgi_request_push_stdin - (struct fcgi_request *self, const void *data, size_t len) -{ - if (self->state != FCGI_REQUEST_STDIN) - { - // TODO: probably reject the request - return; - } - - self->muxer->request_push_cb (self, data, len); -} - static void fcgi_request_flush (struct fcgi_request *self) { @@ -294,36 +262,62 @@ fcgi_request_finish (struct fcgi_request *self) self->muxer->requests[self->request_id] = NULL; fcgi_request_destroy (self); - // TODO: tear down (shut down) the connection. This is called from: - // - // 1. client_fcgi_request_push <- request_push_cb - // <- fcgi_request_push_stdin <- fcgi_muxer_on_stdin - // <- fcgi_muxer_on_message <- fcgi_parser_push <- fcgi_muxer_push - // <- client_fcgi_push <- client_read_loop - // => in this case no close_cb may be called - // -> need to pass a false boolean aaall the way up, - // then client_fcgi_finalize eventually cleans up the rest - // - // 2. client_fcgi_request_close_cb <- request_finish - // => our direct caller must call fcgi_muxer::close_cb - // -> not very nice to delegate it there return !should_close; } +static bool +fcgi_request_push_params + (struct fcgi_request *self, const void *data, size_t len) +{ + if (self->state != FCGI_REQUEST_PARAMS) + { + print_debug ("FastCGI: expected %s, got %s", + STRINGIFY (FCGI_STDIN), STRINGIFY (FCGI_PARAMS)); + return false; + } + + if (len) + fcgi_nv_parser_push (&self->hdr_parser, data, len); + else + { + if (self->hdr_parser.state != FCGI_NV_PARSER_NAME_LEN) + print_debug ("FastCGI: request headers seem to be cut off"); + + self->state = FCGI_REQUEST_STDIN; + if (!self->muxer->request_start_cb (self)) + return fcgi_request_finish (self); + } + return true; +} + +static bool +fcgi_request_push_stdin + (struct fcgi_request *self, const void *data, size_t len) +{ + if (self->state != FCGI_REQUEST_STDIN) + { + print_debug ("FastCGI: expected %s, got %s", + STRINGIFY (FCGI_PARAMS), STRINGIFY (FCGI_STDIN)); + return false; + } + + return self->muxer->request_push_cb (self, data, len); +} + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -typedef void (*fcgi_muxer_handler_fn) +typedef bool (*fcgi_muxer_handler_fn) (struct fcgi_muxer *, const struct fcgi_parser *); -static void +static bool fcgi_muxer_on_get_values (struct fcgi_muxer *self, const struct fcgi_parser *parser) { if (parser->request_id != FCGI_NULL_REQUEST_ID) { - print_debug ("FastCGI: ignoring invalid %s message", + print_debug ("FastCGI: invalid %s message", STRINGIFY (FCGI_GET_VALUES)); - return; + return false; } struct str_map values = str_map_make (free); @@ -357,9 +351,10 @@ fcgi_muxer_on_get_values str_map_free (&values); str_map_free (&response); + return true; } -static void +static bool fcgi_muxer_on_begin_request (struct fcgi_muxer *self, const struct fcgi_parser *parser) { @@ -375,16 +370,17 @@ fcgi_muxer_on_begin_request if (!success) { - print_debug ("FastCGI: ignoring invalid %s message", + print_debug ("FastCGI: invalid %s message", STRINGIFY (FCGI_BEGIN_REQUEST)); - return; + return false; } struct fcgi_request *request = self->requests[parser->request_id]; if (parser->request_id == FCGI_NULL_REQUEST_ID || request) { - // TODO: fail - return; + print_debug ("FastCGI: unusable request ID in %s message", + STRINGIFY (FCGI_BEGIN_REQUEST)); + return false; } // We can only act as a responder, reject everything else up front @@ -392,7 +388,7 @@ fcgi_muxer_on_begin_request { fcgi_muxer_send_end_request (self, parser->request_id, 0, FCGI_UNKNOWN_ROLE); - return; + return true; } if (parser->request_id >= N_ELEMENTS (self->requests) @@ -400,7 +396,7 @@ fcgi_muxer_on_begin_request { fcgi_muxer_send_end_request (self, parser->request_id, 0, FCGI_OVERLOADED); - return; + return true; } request = fcgi_request_new (); @@ -410,9 +406,10 @@ fcgi_muxer_on_begin_request self->requests[parser->request_id] = request; self->active_requests++; + return true; } -static void +static bool fcgi_muxer_on_abort_request (struct fcgi_muxer *self, const struct fcgi_parser *parser) { @@ -421,15 +418,13 @@ fcgi_muxer_on_abort_request { print_debug ("FastCGI: received %s for an unknown request", STRINGIFY (FCGI_ABORT_REQUEST)); - return; + return true; // We might have just rejected it } - // TODO: abort the request: let it somehow produce FCGI_END_REQUEST, - // make sure to send an stdout EOF record - // TODO: and if that was not a FCGI_KEEP_CONN request, close the transport + return fcgi_request_finish (request); } -static void +static bool fcgi_muxer_on_params (struct fcgi_muxer *self, const struct fcgi_parser *parser) { struct fcgi_request *request = self->requests[parser->request_id]; @@ -437,14 +432,15 @@ fcgi_muxer_on_params (struct fcgi_muxer *self, const struct fcgi_parser *parser) { print_debug ("FastCGI: received %s for an unknown request", STRINGIFY (FCGI_PARAMS)); - return; + return true; // We might have just rejected it } - fcgi_request_push_params (request, + // This may immediately finish and delete the request, but that's fine + return fcgi_request_push_params (request, parser->content.str, parser->content.len); } -static void +static bool fcgi_muxer_on_stdin (struct fcgi_muxer *self, const struct fcgi_parser *parser) { struct fcgi_request *request = self->requests[parser->request_id]; @@ -452,15 +448,15 @@ fcgi_muxer_on_stdin (struct fcgi_muxer *self, const struct fcgi_parser *parser) { print_debug ("FastCGI: received %s for an unknown request", STRINGIFY (FCGI_STDIN)); - return; + return true; // We might have just rejected it } // At the end of the stream, a zero-length record is received - fcgi_request_push_stdin (request, + return fcgi_request_push_stdin (request, parser->content.str, parser->content.len); } -static void +static bool fcgi_muxer_on_message (const struct fcgi_parser *parser, void *user_data) { struct fcgi_muxer *self = user_data; @@ -468,8 +464,7 @@ fcgi_muxer_on_message (const struct fcgi_parser *parser, void *user_data) if (parser->version != FCGI_VERSION_1) { print_debug ("FastCGI: unsupported version %d", parser->version); - // TODO: also return false to stop processing on protocol error? - return; + return false; } static const fcgi_muxer_handler_fn handlers[] = @@ -489,10 +484,10 @@ fcgi_muxer_on_message (const struct fcgi_parser *parser, void *user_data) uint8_t content[8] = { parser->type }; fcgi_muxer_send (self, FCGI_UNKNOWN_TYPE, parser->request_id, content, sizeof content); - return; + return true; } - handler (self, parser); + return handler (self, parser); } static void @@ -520,10 +515,10 @@ fcgi_muxer_free (struct fcgi_muxer *self) fcgi_parser_free (&self->parser); } -static void +static bool fcgi_muxer_push (struct fcgi_muxer *self, const void *data, size_t len) { - fcgi_parser_push (&self->parser, data, len); + return fcgi_parser_push (&self->parser, data, len); } /// @} @@ -1574,7 +1569,7 @@ struct request /// Callback to write some CGI response data to the output void (*write_cb) (struct request *, const void *data, size_t len); - /// Callback to close the connection. + /// Callback to close the CGI response, simulates end of program execution. /// CALLING THIS MAY CAUSE THE REQUEST TO BE DESTROYED. void (*close_cb) (struct request *); }; @@ -1967,7 +1962,7 @@ client_destroy (struct client *self) static void client_write (struct client *self, const void *data, size_t len) { - if (!soft_assert (!self->flushing)) + if (!soft_assert (!self->flushing) || len == 0) return; struct write_req *req = xcalloc (1, sizeof *req); @@ -2206,8 +2201,7 @@ static bool client_fcgi_push (struct client *client, const void *data, size_t len) { FIND_CONTAINER (self, client, struct client_fcgi, client); - fcgi_muxer_push (&self->muxer, data, len); - return true; + return fcgi_muxer_push (&self->muxer, data, len); } static void diff --git a/liberty b/liberty index 9494e8e..bca7167 160000 --- a/liberty +++ b/liberty @@ -1 +1 @@ -Subproject commit 9494e8e2affcd08b791d3ecf68985a8a3a310e55 +Subproject commit bca7167d037d857448cb18243425d7c61de3bdd5 From 8d664355681302a08b671723699e3b3720a7703a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Thu, 18 Oct 2018 06:41:12 +0200 Subject: [PATCH 77/79] Remember to set the server context in SCGI requests --- demo-json-rpc-server.c | 1 + 1 file changed, 1 insertion(+) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index 50e203f..e210788 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -2339,6 +2339,7 @@ client_scgi_create (EV_P_ int sock_fd) self->client.vtable = &client_scgi_vtable; request_init (&self->request); + self->request.ctx = ev_userdata (EV_DEFAULT); self->request.write_cb = client_scgi_write_cb; self->request.close_cb = client_scgi_close_cb; From 6e152ae37c4075ff2a39298d450bf58258ceffe1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Thu, 18 Oct 2018 07:17:06 +0200 Subject: [PATCH 78/79] More debugging information for static file serving --- demo-json-rpc-server.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/demo-json-rpc-server.c b/demo-json-rpc-server.c index e210788..34cd378 100644 --- a/demo-json-rpc-server.c +++ b/demo-json-rpc-server.c @@ -1642,6 +1642,15 @@ request_start (struct request *self, struct str_map *headers) // Another way we could get rid of the continue_ argument is via adding // some way of marking the request as finished from within the handler. + if (g_debug_mode) + { + struct str_map_iter iter = str_map_iter_make (headers); + const char *value; + while ((value = str_map_iter_next (&iter))) + print_debug ("%s: %s", iter.link->key, value); + print_debug ("--"); + } + bool continue_ = true; LIST_FOR_EACH (struct request_handler, handler, self->ctx->handlers) if (handler->try_handle (self, headers, &continue_)) @@ -1800,15 +1809,19 @@ request_handler_static_try_handle return false; } + // TODO: implement HEAD, we don't get that for free; + // probably implies adding Content-Length const char *method = str_map_find (headers, "REQUEST_METHOD"); if (!method || strcmp (method, "GET")) return false; // TODO: look at , REQUEST_URI in the headers const char *path_info = str_map_find (headers, "PATH_INFO"); + if (!path_info) + path_info = str_map_find (headers, "REQUEST_URI"); if (!path_info) { - print_debug ("PATH_INFO not defined"); + print_debug ("neither PATH_INFO nor REQUEST_URI was defined"); return false; } @@ -1816,6 +1829,7 @@ request_handler_static_try_handle // Being able to read /etc/passwd would be rather embarrasing char *suffix = canonicalize_url_path (path_info); char *path = xstrdup_printf ("%s%s", root, suffix); + print_debug ("trying to statically serve %s", path); // TODO: check that this is a regular file FILE *fp = fopen (path, "rb"); From 1c52f9e37ed07e93c21f84779d209ee196360006 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Janouch?= Date: Thu, 18 Oct 2018 07:27:55 +0200 Subject: [PATCH 79/79] demo-json-rpc-server -> json-rpc-test-server --- CMakeLists.txt | 6 +++--- demo-json-rpc-server.c => json-rpc-test-server.c | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) rename demo-json-rpc-server.c => json-rpc-test-server.c (99%) diff --git a/CMakeLists.txt b/CMakeLists.txt index c78736a..f6d9b00 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,9 +36,9 @@ configure_file (${PROJECT_SOURCE_DIR}/config.h.in ${PROJECT_BINARY_DIR}/config.h include_directories (${PROJECT_BINARY_DIR}) # Build the executables -add_executable (demo-json-rpc-server - demo-json-rpc-server.c http-parser/http_parser.c) -target_link_libraries (demo-json-rpc-server ${project_libraries}) +add_executable (json-rpc-test-server + json-rpc-test-server.c http-parser/http_parser.c) +target_link_libraries (json-rpc-test-server ${project_libraries}) # The files to be installed include (GNUInstallDirs) diff --git a/demo-json-rpc-server.c b/json-rpc-test-server.c similarity index 99% rename from demo-json-rpc-server.c rename to json-rpc-test-server.c index 34cd378..ac15f0a 100644 --- a/demo-json-rpc-server.c +++ b/json-rpc-test-server.c @@ -1,5 +1,5 @@ /* - * demo-json-rpc-server.c: JSON-RPC 2.0 demo server + * json-rpc-test-server.c: JSON-RPC 2.0 demo server * * Copyright (c) 2015 - 2018, Přemysl Janouch *