Bump version, update NEWS

Sadly, the 3.5 deprecation warning doesn't go away after this.

.clang-format

@@ -0,0 +1,33 @@
+# clang-format is fairly limited, and these rules are approximate:
+#  - array initializers can get terribly mangled with clang-format 12.0,
+#  - sometimes it still aligns with space characters,
+#  - EV_DEFAULT_ and EV_A_ are always taken as identifiers,
+#  - struct name NL { NL ... NL } NL name; is unachievable.
+BasedOnStyle: GNU
+ColumnLimit: 80
+IndentWidth: 4
+TabWidth: 4
+UseTab: ForContinuationAndIndentation
+BreakBeforeBraces: Allman
+SpaceAfterCStyleCast: true
+AlignAfterOpenBracket: DontAlign
+AlignOperands: DontAlign
+AlignConsecutiveMacros: Consecutive
+AllowAllArgumentsOnNextLine: false
+AllowAllParametersOfDeclarationOnNextLine: false
+IndentGotoLabels: false
+
+# IncludeCategories has some potential, but it may also break the build.
+# Note that the documentation says the value should be "Never".
+SortIncludes: false
+
+# This is a compromise, it generally works out aesthetically better.
+BinPackArguments: false
+
+# Unfortunately, this can't be told to align to column 40 or so.
+SpacesBeforeTrailingComments: 2
+
+# liberty-specific macro body wrappers.
+MacroBlockBegin: "BLOCK_START"
+MacroBlockEnd: "BLOCK_END"
+ForEachMacros: ["LIST_FOR_EACH"]

.gitignore

@@ -7,3 +7,5 @@
 /json-rpc-shell.files
 /json-rpc-shell.creator*
 /json-rpc-shell.includes
+/json-rpc-shell.cflags
+/json-rpc-shell.cxxflags

CMakeLists.txt

@@ -1,5 +1,5 @@
-project (json-rpc-shell C)
+cmake_minimum_required (VERSION 3.0...3.27)
-cmake_minimum_required (VERSION 2.8.5)
+project (json-rpc-shell VERSION 1.2.0 LANGUAGES C)
 
 # Options
 option (WANT_READLINE "Use GNU Readline for the UI (better)" ON)
@@ -10,54 +10,73 @@ if ("${CMAKE_C_COMPILER_ID}" MATCHES "GNU" OR CMAKE_COMPILER_IS_GNUCC)
 	# -Wunused-function is pretty annoying here, as everything is static
 	set (CMAKE_C_FLAGS
 		"${CMAKE_C_FLAGS} -std=c99 -Wall -Wextra -Wno-unused-function")
-endif ("${CMAKE_C_COMPILER_ID}" MATCHES "GNU" OR CMAKE_COMPILER_IS_GNUCC)
+endif ()
-
-# Version
-set (project_VERSION_MAJOR "1")
-set (project_VERSION_MINOR "1")
-set (project_VERSION_PATCH "0")
-
-set (project_VERSION "${project_VERSION_MAJOR}")
-set (project_VERSION "${project_VERSION}.${project_VERSION_MINOR}")
-set (project_VERSION "${project_VERSION}.${project_VERSION_PATCH}")
 
 # For custom modules
-set (CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake)
+set (CMAKE_MODULE_PATH
+	"${PROJECT_SOURCE_DIR}/cmake;${PROJECT_SOURCE_DIR}/liberty/cmake")
 
 # Dependencies
 find_package (Curses)
+find_package (Ncursesw)
 find_package (PkgConfig REQUIRED)
-pkg_check_modules (dependencies REQUIRED libcurl jansson)
 # Note that cURL can link to a different version of libssl than we do,
 # in which case the results are undefined
-pkg_check_modules (libssl REQUIRED libssl libcrypto)
+pkg_check_modules (dependencies REQUIRED libcurl jansson libssl libcrypto)
 find_package (LibEV REQUIRED)
-pkg_check_modules (ncursesw ncursesw)
 
-set (project_libraries ${dependencies_LIBRARIES}
+set (project_libraries ${dependencies_LIBRARIES} ${LibEV_LIBRARIES})
-	${libssl_LIBRARIES} ${LIBEV_LIBRARIES})
+include_directories (${dependencies_INCLUDE_DIRS} ${LibEV_INCLUDE_DIRS})
-include_directories (${dependencies_INCLUDE_DIRS}
+link_directories (${dependencies_LIBRARY_DIRS})
-	${libssl_INCLUDE_DIRS} ${LIBEV_INCLUDE_DIRS})
 
-if (ncursesw_FOUND)
+if ("${CMAKE_SYSTEM_NAME}" MATCHES "BSD")
-	list (APPEND project_libraries ${ncursesw_LIBRARIES})
+	# Need this for SIGWINCH in FreeBSD and OpenBSD respectively;
-	include_directories (${ncursesw_INCLUDE_DIRS})
+	# our POSIX version macros make it undefined
+	add_definitions (-D__BSD_VISIBLE=1 -D_BSD_SOURCE=1)
+elseif (APPLE)
+	add_definitions (-D_DARWIN_C_SOURCE)
+endif ()
+
+# -liconv may or may not be a part of libc
+find_library (iconv_LIBRARIES iconv)
+if (iconv_LIBRARIES)
+	list (APPEND project_libraries ${iconv_LIBRARIES})
+endif ()
+
+include (CheckCSourceRuns)
+set (CMAKE_REQUIRED_LIBRARIES ${project_libraries})
+get_property (CMAKE_REQUIRED_INCLUDES
+	DIRECTORY "${PROJECT_SOURCE_DIR}" PROPERTY INCLUDE_DIRECTORIES)
+CHECK_C_SOURCE_RUNS ("#include <iconv.h>
+	int main () { return iconv_open (\"UTF-8//TRANSLIT\", \"ISO-8859-1\")
+		== (iconv_t) -1; }" ICONV_ACCEPTS_TRANSLIT)
+
+if (Ncursesw_FOUND)
+	list (APPEND project_libraries ${Ncursesw_LIBRARIES})
+	include_directories (${Ncursesw_INCLUDE_DIRS})
+	link_directories (${Ncursesw_LIBRARY_DIRS})
 elseif (CURSES_FOUND)
 	list (APPEND project_libraries ${CURSES_LIBRARY})
 	include_directories (${CURSES_INCLUDE_DIR})
-else (CURSES_FOUND)
+else ()
 	message (SEND_ERROR "Curses not found")
-endif (ncursesw_FOUND)
+endif ()
 
 if ((WANT_READLINE AND WANT_LIBEDIT) OR (NOT WANT_READLINE AND NOT WANT_LIBEDIT))
 	message (SEND_ERROR "You have to choose either GNU Readline or libedit")
 elseif (WANT_READLINE)
-	list (APPEND project_libraries readline)
+	# OpenBSD's default readline is too old
+	if ("${CMAKE_SYSTEM_NAME}" MATCHES "OpenBSD")
+		include_directories (${OPENBSD_LOCALBASE}/include/ereadline)
+		list (APPEND project_libraries ereadline)
+	else ()
+		list (APPEND project_libraries readline)
+	endif ()
 elseif (WANT_LIBEDIT)
 	pkg_check_modules (libedit REQUIRED libedit)
 	list (APPEND project_libraries ${libedit_LIBRARIES})
 	include_directories (${libedit_INCLUDE_DIRS})
-endif ((WANT_READLINE AND WANT_LIBEDIT) OR (NOT WANT_READLINE AND NOT WANT_LIBEDIT))
+endif ()
 
 # Generate a configuration file
 set (HAVE_READLINE "${WANT_READLINE}")
@@ -86,23 +105,43 @@ install (TARGETS ${PROJECT_NAME} DESTINATION ${CMAKE_INSTALL_BINDIR})
 install (PROGRAMS json-format.pl DESTINATION ${CMAKE_INSTALL_BINDIR})
 install (FILES LICENSE DESTINATION ${CMAKE_INSTALL_DOCDIR})
 
-# Generate documentation from program help
+# Generate documentation from text markup
 find_program (ASCIIDOCTOR_EXECUTABLE asciidoctor)
-if (NOT ASCIIDOCTOR_EXECUTABLE)
+find_program (A2X_EXECUTABLE a2x)
-	message (FATAL_ERROR "asciidoctor not found")
+if (NOT ASCIIDOCTOR_EXECUTABLE AND NOT A2X_EXECUTABLE)
-endif (NOT ASCIIDOCTOR_EXECUTABLE)
+	message (WARNING "Neither asciidoctor nor a2x were found, "
+		"falling back to a substandard manual page generator")
+endif ()
 
 foreach (page ${PROJECT_NAME})
 	set (page_output "${PROJECT_BINARY_DIR}/${page}.1")
 	list (APPEND project_MAN_PAGES "${page_output}")
-	add_custom_command (OUTPUT ${page_output}
+	if (ASCIIDOCTOR_EXECUTABLE)
-		COMMAND ${ASCIIDOCTOR_EXECUTABLE} -b manpage
+		add_custom_command (OUTPUT ${page_output}
-			-a release-version=${project_VERSION}
+			COMMAND ${ASCIIDOCTOR_EXECUTABLE} -b manpage
-			"${PROJECT_SOURCE_DIR}/${page}.adoc"
+				-a release-version=${PROJECT_VERSION}
-			-o "${page_output}"
+				-o "${page_output}"
-		DEPENDS ${page}.adoc
+				"${PROJECT_SOURCE_DIR}/${page}.adoc"
-		COMMENT "Generating man page for ${page}" VERBATIM)
+			DEPENDS ${page}.adoc
-endforeach (page)
+			COMMENT "Generating man page for ${page}" VERBATIM)
+	elseif (A2X_EXECUTABLE)
+		add_custom_command (OUTPUT ${page_output}
+			COMMAND ${A2X_EXECUTABLE} --doctype manpage --format manpage
+				-a release-version=${PROJECT_VERSION}
+				-D "${PROJECT_BINARY_DIR}"
+				"${PROJECT_SOURCE_DIR}/${page}.adoc"
+			DEPENDS ${page}.adoc
+			COMMENT "Generating man page for ${page}" VERBATIM)
+	else ()
+		set (ASCIIMAN ${PROJECT_SOURCE_DIR}/liberty/tools/asciiman.awk)
+		add_custom_command (OUTPUT ${page_output}
+			COMMAND env LC_ALL=C asciidoc-release-version=${PROJECT_VERSION}
+				awk -f ${ASCIIMAN} "${PROJECT_SOURCE_DIR}/${page}.adoc"
+				> ${page_output}
+			DEPENDS ${page}.adoc ${ASCIIMAN}
+			COMMENT "Generating man page for ${page}" VERBATIM)
+	endif ()
+endforeach ()
 
 add_custom_target (docs ALL DEPENDS ${project_MAN_PAGES})
 
@@ -110,22 +149,19 @@ foreach (page ${project_MAN_PAGES})
 	string (REGEX MATCH "\\.([0-9])$" manpage_suffix "${page}")
 	install (FILES "${page}"
 		DESTINATION "${CMAKE_INSTALL_MANDIR}/man${CMAKE_MATCH_1}")
-endforeach (page)
+endforeach ()
 
 # CPack
 set (CPACK_PACKAGE_DESCRIPTION_SUMMARY "A shell for JSON-RPC 2.0")
 set (CPACK_PACKAGE_VENDOR "Premysl Eric Janouch")
 set (CPACK_PACKAGE_CONTACT "Přemysl Eric Janouch <p@janouch.name>")
 set (CPACK_RESOURCE_FILE_LICENSE "${PROJECT_SOURCE_DIR}/LICENSE")
-set (CPACK_PACKAGE_VERSION_MAJOR ${project_VERSION_MAJOR})
-set (CPACK_PACKAGE_VERSION_MINOR ${project_VERSION_MINOR})
-set (CPACK_PACKAGE_VERSION_PATCH ${project_VERSION_PATCH})
 set (CPACK_GENERATOR "TGZ;ZIP")
 set (CPACK_PACKAGE_FILE_NAME
-	"${PROJECT_NAME}-${project_VERSION}-${CMAKE_SYSTEM_NAME}-${CMAKE_SYSTEM_PROCESSOR}")
+	"${PROJECT_NAME}-${PROJECT_VERSION}-${CMAKE_SYSTEM_NAME}-${CMAKE_SYSTEM_PROCESSOR}")
-set (CPACK_PACKAGE_INSTALL_DIRECTORY "${PROJECT_NAME}-${project_VERSION}")
+set (CPACK_PACKAGE_INSTALL_DIRECTORY "${PROJECT_NAME}-${PROJECT_VERSION}")
 set (CPACK_SOURCE_GENERATOR "TGZ;ZIP")
 set (CPACK_SOURCE_IGNORE_FILES "/\\\\.git;/build;/CMakeLists.txt.user")
-set (CPACK_SOURCE_PACKAGE_FILE_NAME "${PROJECT_NAME}-${project_VERSION}")
+set (CPACK_SOURCE_PACKAGE_FILE_NAME "${PROJECT_NAME}-${PROJECT_VERSION}")
 
 include (CPack)

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2014 - 2020, Přemysl Eric Janouch <p@janouch.name>
+Copyright (c) 2014 - 2022, Přemysl Eric Janouch <p@janouch.name>
 
 Permission to use, copy, modify, and/or distribute this software for any
 purpose with or without fee is hereby granted.

NEWS

@@ -1,3 +1,22 @@
+1.2.0 (2024-12-24)
+
+ * Add a backend for co-processes, such as language servers
+
+ * Support reading OpenRPC documents from a file
+
+ * Respect the NO_COLOR environment variable
+
+ * Miscellaneous libedit (editline) fixes
+
+ * Miscellaneous portability improvements
+
+ * json-rpc-test-server: implement OpenRPC discovery
+
+ * json-rpc-test-server: only serve regular files
+
+ * json-rpc-test-server: miscellaneous WebSocket fixes
+
+
 1.1.0 (2020-10-13)
 
  * Add method name tab completion using OpenRPC information

README.adoc

@@ -5,7 +5,7 @@ json-rpc-shell
 'json-rpc-shell' is a shell for running JSON-RPC 2.0 queries.
 
 This software was originally created as a replacement for
-http://software.dzhuvinov.com/json-rpc-2.0-shell.html[a different one] made by
+http://software.dzhuvinov.com/json-rpc-2.0-shell.html[a different shell] made by
 Vladimir Dzhuvinov, in order to avoid Java, but has evolved since.
 
 Features
@@ -29,19 +29,17 @@ The rest of this README will concern itself with externalities.
 
 Packages
 --------
-Regular releases are sporadic.  git master should be stable enough.  You can get
+Regular releases are sporadic.  git master should be stable enough.
-a package with the latest development version from Archlinux's AUR.
+You can get a package with the latest development version using Arch Linux's
+https://aur.archlinux.org/packages/json-rpc-shell-git[AUR],
+or as a https://git.janouch.name/p/nixexprs[Nix derivation].
 
 Building
 --------
-Build dependencies: CMake, pkg-config, asciidoctor,
+Build dependencies: CMake, pkg-config, liberty (included),
-                    liberty (included), http-parser (included) +
+ http-parser (included), asciidoctor or asciidoc (recommended but optional) +
-Runtime dependencies: libev, Jansson, cURL, openssl,
+Runtime dependencies:
-                      readline or libedit >= 2013-07-12,
+ libev, Jansson, cURL, openssl, readline or libedit >= 2013-07-12
-
-Avoid libedit if you can, in general it works but at the moment history is
-acting up and I have no clue about fixing it.  Multiline editing is also
-misbehaving there.
 
  $ git clone --recursive https://git.janouch.name/p/json-rpc-shell.git
  $ mkdir json-rpc-shell/build
@@ -58,15 +56,12 @@ Or you can try telling CMake to make a package for you.  For Debian it is:
  $ cpack -G DEB
  # dpkg -i json-rpc-shell-*.deb
 
-Note that for versions of CMake before 2.8.9, you need to prefix `cpack` with
-`fakeroot` or file ownership will end up wrong.
-
 Test server
 -----------
 If you install development packages for libmagic, an included test server will
 be built but not installed which provides a trivial JSON-RPC 2.0 service with
-FastCGI, SCGI, and WebSocket interfaces.  It responds to `ping` and `date`
+FastCGI, SCGI, WebSocket and LSP-like co-process interfaces.  It responds to
-methods and it can serve static files.
+`ping` and `date`, supports OpenRPC discovery and it can serve static files.
 
 Contributing and Support
 ------------------------

cmake/FindLibEV.cmake

@@ -5,14 +5,16 @@
 
 # Some distributions do add it, though
 find_package (PkgConfig REQUIRED)
-pkg_check_modules (LIBEV QUIET libev)
+pkg_check_modules (LibEV QUIET libev)
 
-if (NOT LIBEV_FOUND)
+set (required_vars LibEV_LIBRARIES)
-	find_path (LIBEV_INCLUDE_DIRS ev.h)
+if (NOT LibEV_FOUND)
-	find_library (LIBEV_LIBRARIES NAMES ev)
+	find_path (LibEV_INCLUDE_DIRS ev.h)
+	find_library (LibEV_LIBRARIES NAMES ev)
+	list (APPEND required_vars LibEV_INCLUDE_DIRS)
+endif ()
 
-	if (LIBEV_INCLUDE_DIRS AND LIBEV_LIBRARIES)
+include (FindPackageHandleStandardArgs)
-		set (LIBEV_FOUND TRUE)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS (LibEV DEFAULT_MSG ${required_vars})
-	endif (LIBEV_INCLUDE_DIRS AND LIBEV_LIBRARIES)
-endif (NOT LIBEV_FOUND)
 
+mark_as_advanced (LibEV_LIBRARIES LibEV_INCLUDE_DIRS)

config.h.in

@@ -2,10 +2,12 @@
 #define CONFIG_H
 
 #define PROGRAM_NAME "${PROJECT_NAME}"
-#define PROGRAM_VERSION "${project_VERSION}"
+#define PROGRAM_VERSION "${PROJECT_VERSION}"
 
 #cmakedefine HAVE_READLINE
 #cmakedefine HAVE_EDITLINE
 
+#cmakedefine01 ICONV_ACCEPTS_TRANSLIT
+
 #endif  // ! CONFIG_H
 

json-rpc-shell.adoc

@@ -14,8 +14,9 @@ Synopsis
 
 Description
 -----------
+:colon: :
 The _ENDPOINT_ must be either an HTTP or a WebSocket URL, with or without TLS
-(i.e. one of the _+++http+++://_, _+++https+++://_, _ws://_, _wss://_ schemas).
+(i.e. one of the _http{colon}//_, _https{colon}//_, _ws://_, _wss://_ schemas).
 
 *json-rpc-shell* will use it to send any JSON-RPC 2.0 requests you enter on its
 command line. The server's response will be parsed and validated, stripping it
@@ -96,8 +97,16 @@ Program information
 *--write-default-cfg*[**=**__PATH__]::
 	Write a default configuration file, show its path and exit.
 
+Environment
+-----------
+*VISUAL*, *EDITOR*::
+	The editor program to be launched by the M-e key binding.
+	If neither variable is set, it defaults to *vi*(1).
+
 Files
 -----
+*json-rpc-shell* follows the XDG Base Directory Specification.
+
 _~/.config/json-rpc-shell/json-rpc-shell.conf_::
 	The configuration file, in which you can configure color output and
 	CA certificate paths.  Use the *--write-default-cfg* option to create
@@ -132,8 +141,7 @@ the higher-level protocol (the "Sec-Ws-Protocol" HTTP field).
 
 Bugs
 ----
-The editline (libedit) frontend is more of a proof of concept that mostly seems
+The editline (libedit) frontend may exhibit some unexpected behaviour.
-to work but exhibits bugs that are not our fault.
 
 Examples
 --------

json-rpc-shell.c

@@ -1,7 +1,7 @@
 /*
  * json-rpc-shell.c: a shell for JSON-RPC 2.0
  *
- * Copyright (c) 2014 - 2020, Přemysl Eric Janouch <p@janouch.name>
+ * Copyright (c) 2014 - 2022, Přemysl Eric Janouch <p@janouch.name>
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted.
@@ -515,6 +515,7 @@ struct input_el
 	char *entered_line;                 ///< Buffers the entered line
 
 	bool active;                        ///< Interface has been started
+	bool need_restart;                  ///< Need to clear history state
 	char *prompt;                       ///< The prompt we use
 	int prompt_shown;                   ///< Whether the prompt is shown now
 
@@ -535,27 +536,29 @@ input_el_wcstombs (const wchar_t *s)
 	return mb;
 }
 
-static int
-input_el_get_termios (int character, int fallback)
-{
-	if (!g_terminal.initialized)
-		return fallback;
-
-	cc_t value = g_terminal.termios.c_cc[character];
-	if (value == _POSIX_VDISABLE)
-		return fallback;
-	return value;
-}
-
 static void
 input_el_redisplay (struct input_el *self)
 {
-	char x[] = { input_el_get_termios (VREPRINT, 'R' - 0x40), 0 };
+	// See rl_redisplay(), however NetBSD editline's map.c v1.54 breaks VREPRINT
-	el_push (self->editline, x);
+	// so we bind redisplay somewhere else in input_el_start()
+	wchar_t x[] = { L'q' & 31, 0 };
+	el_wpush (self->editline, x);
 
 	// We have to do this or it gets stuck and nothing is done
-	int count = 0;
+	int dummy_count = 0;
-	(void) el_wgets (self->editline, &count);
+	(void) el_wgets (self->editline, &dummy_count);
+}
+
+// Editline keeping its own history position (look for "eventno" there).
+// This is the only sane way of resetting it.
+static void
+input_el_start_over (struct input_el *self)
+{
+	wchar_t x[] = { L'c' & 31, 0 };
+	el_wpush (self->editline, x);
+
+	int dummy_count = 0;
+	(void) el_wgets (self->editline, &dummy_count);
 }
 
 static char *
@@ -598,7 +601,7 @@ input_el_on_return (EditLine *editline, int key)
 	int len = info->lastchar - info->buffer;
 	int point = info->cursor - info->buffer;
 
-	wchar_t *line = calloc (sizeof *info->buffer, len + 1);
+	wchar_t *line = xcalloc (len + 1, sizeof *info->buffer);
 	memcpy (line, info->buffer, sizeof *info->buffer * len);
 
 	if (*line)
@@ -613,13 +616,14 @@ input_el_on_return (EditLine *editline, int key)
 	self->entered_line = xstrndup
 		(info_mb->buffer, info_mb->lastchar - info_mb->buffer);
 
-	// Now we need to force editline to actually print the newline
+	// Now we need to force editline into actually printing the newline
 	el_cursor (editline, len++ - point);
 	el_insertstr (editline, "\n");
 	input_el_redisplay (self);
 
 	// Finally we need to discard the old line's contents
 	el_wdeletestr (editline, len);
+	self->need_restart = true;
 	return CC_NEWLINE;
 }
 
@@ -698,6 +702,11 @@ input_el_start (struct input *input, const char *program_name)
 	// Source the user's defaults file
 	el_source (self->editline, NULL);
 
+	// See input_el_redisplay(), functionally important
+	el_set (self->editline, EL_BIND, "^q", "ed-redisplay",        NULL);
+	// This is what buffered el_wgets() does, functionally important
+	el_set (self->editline, EL_BIND, "^c", "ed-start-over",       NULL);
+
 	self->active = true;
 	self->prompt_shown = 1;
 }
@@ -753,7 +762,7 @@ input_el_hide (struct input *input)
 	int len = info->lastchar - info->buffer;
 	int point = info->cursor - info->buffer;
 
-	wchar_t *line = calloc (sizeof *info->buffer, len + 1);
+	wchar_t *line = xcalloc (len + 1, sizeof *info->buffer);
 	memcpy (line, info->buffer, sizeof *info->buffer * len);
 	el_cursor (self->editline, len - point);
 	el_wdeletestr (self->editline, len);
@@ -985,6 +994,16 @@ input_el_on_tty_readable (struct input *input)
 	// We bind the return key to process it how we need to
 	struct input_el *self = (struct input_el *) input;
 
+	int unbuffered = 0;
+	(void) el_get (self->editline, EL_UNBUFFERED, &unbuffered);
+
+	// We must invoke ch_reset(), which isn't done for us with EL_UNBUFFERED.
+	if (unbuffered && self->need_restart)
+	{
+		self->need_restart = false;
+		input_el_start_over (self);
+	}
+
 	// el_gets() with EL_UNBUFFERED doesn't work with UTF-8,
 	// we must use the wide-character interface
 	int count = 0;
@@ -992,8 +1011,7 @@ input_el_on_tty_readable (struct input *input)
 
 	// Editline works in a funny NO_TTY mode when the input is not a tty,
 	// we cannot use EL_UNBUFFERED and expect sane results then
-	int unbuffered = 0;
+	if (!unbuffered)
-	if (!el_get (self->editline, EL_UNBUFFERED, &unbuffered) && !unbuffered)
 	{
 		char *entered_line = buf ? input_el_wcstombs (buf) : NULL;
 		self->super.on_input (entered_line, self->super.user_data);
@@ -1207,7 +1225,7 @@ await_try_cancel (struct app_context *ctx)
 
 static void on_config_attribute_change (struct config_item *item);
 
-static struct config_schema g_config_connection[] =
+static const struct config_schema g_config_connection[] =
 {
 	{ .name      = "tls_ca_file",
 	  .comment   = "OpenSSL CA bundle file",
@@ -1218,7 +1236,7 @@ static struct config_schema g_config_connection[] =
 	{}
 };
 
-static struct config_schema g_config_attributes[] =
+static const struct config_schema g_config_attributes[] =
 {
 #define XX(x, y, z) { .name = y, .comment = z, .type = CONFIG_ITEM_STRING, \
 	.on_change = on_config_attribute_change },
@@ -1425,7 +1443,7 @@ init_colors (struct app_context *ctx)
 		g_terminal.stderr_is_tty = true;
 		break;
 	case COLOR_AUTO:
-		if (!g_terminal.initialized)
+		if (!g_terminal.initialized || getenv ("NO_COLOR"))
 		{
 	case COLOR_NEVER:
 			g_terminal.stdout_is_tty = false;
@@ -2291,6 +2309,8 @@ backend_ws_connect (struct ws_context *self, struct error **e)
 	str_append_printf (&request, "GET %s HTTP/1.1\r\n", url_path.str);
 	// TODO: omit the port if it's the default (check RFC for "SHOULD" or ...)
 	str_append_printf (&request, "Host: %s:%s\r\n", url_host, url_port);
+	str_append_printf (&request, "User-Agent: %s/%s\r\n",
+		PROGRAM_NAME, PROGRAM_VERSION);
 	str_append_printf (&request, "Upgrade: websocket\r\n");
 	str_append_printf (&request, "Connection: upgrade\r\n");
 	str_append_printf (&request, SEC_WS_KEY ": %s\r\n", key_b64_string);
@@ -2623,7 +2643,7 @@ static const http_parser_settings backend_co_http_settings =
 };
 
 static bool
-backend_co_write_starter (struct co_context *self, struct error **e)
+backend_co_inject_starter (struct co_context *self, struct error **e)
 {
 	// The default "Connection: keep-alive" maps well here.
 	// We cannot feed this line into the parser from within callbacks.
@@ -2653,7 +2673,7 @@ backend_co_parse (struct co_context *self, const char *data, size_t len,
 	if (self->pending_fake_starter)
 	{
 		self->pending_fake_starter = false;
-		if (!backend_co_write_starter (self, e))
+		if (!backend_co_inject_starter (self, e))
 			return false;
 	}
 
@@ -3668,7 +3688,7 @@ complete_method_name (const char *text, int state)
 // --- Main program ------------------------------------------------------------
 
 // The ability to use an external editor on the input line has been shamelessly
-// copypasted from degesch with minor changes only.
+// copypasted from xC with minor changes only.
 
 static bool
 dump_line_to_file (const char *line, char *template, struct error **e)
@@ -4056,11 +4076,10 @@ main (int argc, char *argv[])
 	setlocale (LC_CTYPE, "");
 
 	char *encoding = nl_langinfo (CODESET);
-#ifdef __linux__
 	// XXX: not quite sure if this is actually desirable
 	// TODO: instead retry with JSON_ENSURE_ASCII
-	encoding = xstrdup_printf ("%s//TRANSLIT", encoding);
+	if (ICONV_ACCEPTS_TRANSLIT)
-#endif // __linux__
+		encoding = xstrdup_printf ("%s//TRANSLIT", encoding);
 
 	if ((g_ctx.term_from_utf8 = iconv_open (encoding, "UTF-8"))
 		== (iconv_t) -1

json-rpc-test-server.c

@@ -1,7 +1,7 @@
 /*
  * json-rpc-test-server.c: JSON-RPC 2.0 demo server
  *
- * Copyright (c) 2015 - 2020, Přemysl Eric Janouch <p@janouch.name>
+ * Copyright (c) 2015 - 2022, Přemysl Eric Janouch <p@janouch.name>
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted.
@@ -851,6 +851,17 @@ ws_handler_on_close_timeout (EV_P_ ev_timer *watcher, int revents)
 	self->close_cb (self, false /* half_close */);
 }
 
+static bool ws_handler_fail_handshake (struct ws_handler *self,
+	const char *status, ...) ATTRIBUTE_SENTINEL;
+
+#define HTTP_101_SWITCHING_PROTOCOLS    "101 Switching Protocols"
+#define HTTP_400_BAD_REQUEST            "400 Bad Request"
+#define HTTP_405_METHOD_NOT_ALLOWED     "405 Method Not Allowed"
+#define HTTP_408_REQUEST_TIMEOUT        "408 Request Timeout"
+#define HTTP_417_EXPECTATION_FAILED     "407 Expectation Failed"
+#define HTTP_426_UPGRADE_REQUIRED       "426 Upgrade Required"
+#define HTTP_505_VERSION_NOT_SUPPORTED  "505 HTTP Version Not Supported"
+
 static void
 ws_handler_on_handshake_timeout (EV_P_ ev_timer *watcher, int revents)
 {
@@ -858,13 +869,7 @@ ws_handler_on_handshake_timeout (EV_P_ ev_timer *watcher, int revents)
 	(void) revents;
 	struct ws_handler *self = watcher->data;
 
-	// XXX: this is a no-op, since this currently doesn't even call shutdown
+	ws_handler_fail_handshake (self, HTTP_408_REQUEST_TIMEOUT, NULL);
-	//   immediately but postpones it until later
-	self->close_cb (self, true /* half_close */);
-	self->state = WS_HANDLER_FLUSHING;
-
-	if (self->on_close)
-		self->on_close (self, WS_STATUS_ABNORMAL_CLOSURE, "handshake timeout");
 
 	self->state = WS_HANDLER_CLOSED;
 	self->close_cb (self, false /* half_close */);
@@ -1003,9 +1008,10 @@ ws_handler_on_headers_complete (http_parser *parser)
 	if (self->have_header_value)
 		ws_handler_on_header_read (self);
 
-	// We strictly require a protocol upgrade
+	// We require a protocol upgrade.  1 is for "skip body", 2 is the same
+	// + "stop processing", return another number to indicate a problem here.
 	if (!parser->upgrade)
-		return 2;
+		return 3;
 
 	return 0;
 }
@@ -1018,13 +1024,6 @@ ws_handler_on_url (http_parser *parser, const char *at, size_t len)
 	return 0;
 }
 
-#define HTTP_101_SWITCHING_PROTOCOLS    "101 Switching Protocols"
-#define HTTP_400_BAD_REQUEST            "400 Bad Request"
-#define HTTP_405_METHOD_NOT_ALLOWED     "405 Method Not Allowed"
-#define HTTP_417_EXPECTATION_FAILED     "407 Expectation Failed"
-#define HTTP_426_UPGRADE_REQUIRED       "426 Upgrade Required"
-#define HTTP_505_VERSION_NOT_SUPPORTED  "505 HTTP Version Not Supported"
-
 static void
 ws_handler_http_responsev (struct ws_handler *self,
 	const char *status, char *const *fields)
@@ -1066,6 +1065,7 @@ ws_handler_fail_handshake (struct ws_handler *self, const char *status, ...)
 	struct strv v = strv_make ();
 	while ((s = va_arg (ap, const char *)))
 		strv_append (&v, s);
+	strv_append (&v, "Connection: close");
 
 	va_end (ap);
 	ws_handler_http_responsev (self, status, v.vector);
@@ -1268,11 +1268,13 @@ ws_handler_push (struct ws_handler *self, const void *data, size_t len)
 		ev_timer_stop (EV_DEFAULT_ &self->handshake_timeout_watcher);
 
 		if (err == HPE_CB_headers_complete)
+		{
 			print_debug ("WS handshake failed: %s", "missing `Upgrade' field");
-		else
+			FAIL_HANDSHAKE (HTTP_426_UPGRADE_REQUIRED,
-			print_debug ("WS handshake failed: %s",
+				"Upgrade: websocket", SEC_WS_VERSION ": 13");
-				http_errno_description (err));
+		}
 
+		print_debug ("WS handshake failed: %s", http_errno_description (err));
 		FAIL_HANDSHAKE (HTTP_400_BAD_REQUEST);
 	}
 	return true;
@@ -1462,7 +1464,7 @@ json_rpc_discover (struct server_context *ctx, json_t *params)
 
 	json_t *info = json_pack ("{ssss}",
 		"title", PROGRAM_NAME, "version", PROGRAM_VERSION);
-	json_t *methods = json_pack ("[ooo]",
+	json_t *methods = json_pack ("[oooo]",
 		open_rpc_describe ("date", json_pack ("{ssso}", "type", "object",
 			"properties", json_pack ("{s{ss}s{ss}s{ss}s{ss}s{ss}s{ss}}",
 				"year",    "type", "number",
@@ -1473,7 +1475,8 @@ json_rpc_discover (struct server_context *ctx, json_t *params)
 				"seconds", "type", "number"))),
 		open_rpc_describe ("ping", json_pack ("{ss}", "type", "string")),
 		open_rpc_describe ("rpc.discover", json_pack ("{ss}", "$ref",
-			"https://github.com/open-rpc/meta-schema/raw/master/schema.json")));
+			"https://github.com/open-rpc/meta-schema/raw/master/schema.json")),
+		open_rpc_describe ("wait", json_pack ("{ss}", "type", "null")));
 	return json_rpc_response (NULL, json_pack ("{sssoso}",
 		"openrpc", "1.2.6", "info", info, "methods", methods), NULL);
 }
@@ -1490,6 +1493,16 @@ json_rpc_ping (struct server_context *ctx, json_t *params)
 	return json_rpc_response (NULL, json_string ("pong"), NULL);
 }
 
+static json_t *
+json_rpc_wait (struct server_context *ctx, json_t *params)
+{
+	(void) ctx;
+	(void) params;
+
+	sleep (1);
+	return json_rpc_response (NULL, json_null (), NULL);
+}
+
 static json_t *
 json_rpc_date (struct server_context *ctx, json_t *params)
 {
@@ -1522,6 +1535,7 @@ process_json_rpc_request (struct server_context *ctx, json_t *request)
 		{ "date",         json_rpc_date     },
 		{ "ping",         json_rpc_ping     },
 		{ "rpc.discover", json_rpc_discover },
+		{ "wait",         json_rpc_wait     },
 	};
 
 	if (!json_is_object (request))
@@ -1578,7 +1592,6 @@ static void
 process_json_rpc (struct server_context *ctx,
 	const void *data, size_t len, struct str *output)
 {
-
 	json_error_t e;
 	json_t *request;
 	if (!(request = json_loadb (data, len, JSON_DECODE_ANY, &e)))
@@ -1653,15 +1666,37 @@ struct request_handler
 	LIST_HEADER (struct request_handler)
 
 	/// Install ourselves as the handler for the request, if applicable.
+	/// If the request contains data, check it against CONTENT_LENGTH.
+	/// ("Transfer-Encoding: chunked" should be dechunked by the HTTP server,
+	/// however it is possible that it mishandles this situation.)
 	/// Sets @a continue_ to false if further processing should be stopped,
 	/// meaning the request has already been handled.
+	/// Note that starting the response before receiving all data denies you
+	/// the option of returning error status codes based on the data.
 	bool (*try_handle) (struct request *request,
 		struct str_map *headers, bool *continue_);
 
 	/// Handle incoming data.  "len == 0" means EOF.
 	/// Returns false if there is no more processing to be done.
-	// FIXME: the EOF may or may not be delivered when request is cut short,
+	/// EOF is never delivered on a network error (see client_read_loop()).
-	//   we should fix FastCGI not to deliver it on CONTENT_LENGTH mismatch
+	// XXX: the EOF may or may not be delivered when the request is cut short:
+	//  - client_scgi delivers an EOF when it itself receives an EOF without
+	//    considering any mismatch, and it can deliver another one earlier
+	//    when the counter just goes down to 0... depends on what we return
+	//    from here upon the first occasion (whether we want to close).
+	//  - FCGI_ABORT_REQUEST /might/ not close the stdin and it /might/ cover
+	//    a CONTENT_LENGTH mismatch, since this callback wouldn't get invoked.
+	//    The FastCGI specification explicitly says to compare CONTENT_LENGTH
+	//    against the number of received bytes, which may only be smaller.
+	//
+	// We might want to adjust client_scgi and client_fcgi to not invoke
+	// request_push(EOF) when CONTENT_LENGTH hasn't been reached and remove
+	// the extra EOF generation from client_scgi (why is it there, does the
+	// server keep the connection open, or is it just a precaution?)
+	//
+	// The finalization callback takes care of any needs to destruct data.
+	// If we handle this reliably in all clients, try_handle won't have to,
+	// as it will run in a stricter-than-CGI scenario.
 	bool (*push_cb) (struct request *request, const void *data, size_t len);
 
 	/// Destroy the handler's data stored in the request object
@@ -1783,7 +1818,9 @@ request_handler_json_rpc_push
 
 	// TODO: check buf.len against CONTENT_LENGTH; if it's less, then the
 	//   client hasn't been successful in transferring all of its data.
-	//   See also comment on request_handler::push_cb.
+	//   See also comment on request_handler::push_cb.  For JSON-RPC, though,
+	//   it shouldn't matter as an incomplete request will be invalid and
+	//   clients have no reason to append unnecessary trailing bytes.
 
 	struct str response = str_make ();
 	str_append (&response, "Status: 200 OK\n");
@@ -1900,8 +1937,13 @@ request_handler_static_try_handle
 	char *path = xstrdup_printf ("%s%s", root, suffix);
 	print_debug ("trying to statically serve %s", path);
 
-	// TODO: check that this is a regular file
 	FILE *fp = fopen (path, "rb");
+	struct stat st = {};
+	if (fp && !fstat (fileno (fp), &st) && !S_ISREG (st.st_mode))
+	{
+		fclose (fp);
+		fp = NULL;
+	}
 	if (!fp)
 	{
 		struct str response = str_make ();
@@ -1946,8 +1988,8 @@ request_handler_static_try_handle
 		request_write (request, buf, len);
 	fclose (fp);
 
-	// TODO: this should rather not be returned all at once but in chunks;
+	// TODO: this should rather not be returned all at once but in chunks
-	//   file read requests never return EAGAIN
+	//   (consider Transfer-Encoding); file read requests never return EAGAIN
 	// TODO: actual file data should really be returned by a callback when
 	//   the socket is writable with nothing to be sent (pumping the entire
 	//   file all at once won't really work if it's huge).
@@ -2081,6 +2123,8 @@ static void
 client_shutdown (struct client *self)
 {
 	self->flushing = true;
+	// In case this shutdown is immediately followed by a close, try our best
+	(void) flush_queue (&self->write_queue, self->socket_fd);
 	ev_feed_event (EV_DEFAULT_ &self->write_watcher, EV_WRITE);
 }
 
@@ -2392,14 +2436,15 @@ client_scgi_on_content (void *user_data, const void *data, size_t len)
 		print_debug ("SCGI request got more data than CONTENT_LENGTH");
 		return false;
 	}
-	// We're in a slight disagreement with the specification since
+	// We're in a slight disagreement with the SCGI specification since
 	// this tries to write output before it has read all the input
 	if (!request_push (&self->request, data, len))
 		return false;
+	if ((self->remaining_content -= len))
+		return true;
 
 	// Signalise end of input to the request handler
-	return (self->remaining_content -= len) != 0
+	return request_push (&self->request, NULL, 0);
-		|| request_push (&self->request, NULL, 0);
 }
 
 // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
@@ -2548,6 +2593,165 @@ client_ws_create (EV_P_ int sock_fd)
 	return &self->client;
 }
 
+// --- Co-process client -------------------------------------------------------
+
+// This is mostly copied over from json-rpc-shell.c, only a bit simplified.
+// We're giving up on header parsing in order to keep this small.
+struct co_context
+{
+	struct server_context *ctx;         ///< Server context
+	struct str message;                 ///< Message data
+	struct http_parser parser;          ///< HTTP parser
+	bool pending_fake_starter;          ///< Start of message?
+};
+
+static int
+client_co_on_message_begin (http_parser *parser)
+{
+	struct co_context *self = parser->data;
+	str_reset (&self->message);
+	return 0;
+}
+
+static int
+client_co_on_body (http_parser *parser, const char *at, size_t len)
+{
+	struct co_context *self = parser->data;
+	str_append_data (&self->message, at, len);
+	return 0;
+}
+
+static int
+client_co_on_message_complete (http_parser *parser)
+{
+	struct co_context *self = parser->data;
+	http_parser_pause (&self->parser, true);
+	return 0;
+}
+
+// The LSP incorporates a very thin subset of RFC 822, and it so happens
+// that we may simply reuse the full HTTP parser here, with a small hack.
+static const http_parser_settings client_co_http_settings =
+{
+	.on_message_begin    = client_co_on_message_begin,
+	.on_body             = client_co_on_body,
+	.on_message_complete = client_co_on_message_complete,
+};
+
+static void
+client_co_respond (const struct str *buf)
+{
+	struct str wrapped = str_make();
+	str_append_printf (&wrapped,
+		"Content-Length: %zu\r\n"
+		"Content-Type: application/json; charset=utf-8\r\n"
+		"\r\n", buf->len);
+	str_append_data (&wrapped, buf->str, buf->len);
+
+	if (write (STDOUT_FILENO, wrapped.str, wrapped.len)
+		!= (ssize_t) wrapped.len)
+		exit_fatal ("write: %s", strerror (errno));
+	str_free (&wrapped);
+}
+
+static void
+client_co_inject_starter (struct co_context *self)
+{
+	// The default "Connection: keep-alive" maps well here.
+	// We cannot feed this line into the parser from within callbacks.
+	static const char starter[] = "POST / HTTP/1.1\r\n";
+	http_parser_pause (&self->parser, false);
+
+	size_t n_parsed = http_parser_execute (&self->parser,
+		&client_co_http_settings, starter, sizeof starter - 1);
+	enum http_errno err = HTTP_PARSER_ERRNO (&self->parser);
+	if (n_parsed != sizeof starter - 1 || err != HPE_OK)
+		exit_fatal ("protocol failure: %s", http_errno_description (err));
+}
+
+// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+static void
+client_co_process (struct co_context *self)
+{
+	struct str *message = &self->message;
+	struct str response = str_make ();
+	process_json_rpc (self->ctx, message->str, message->len, &response);
+	if (response.len)
+		client_co_respond (&response);
+	str_free (&response);
+}
+
+static void
+client_co_parse (struct co_context *self, const char *data, size_t len,
+	size_t *n_parsed)
+{
+	if (self->pending_fake_starter)
+	{
+		self->pending_fake_starter = false;
+		client_co_inject_starter (self);
+	}
+
+	*n_parsed = http_parser_execute
+		(&self->parser, &client_co_http_settings, data, len);
+	if (self->parser.upgrade)
+		exit_fatal ("protocol failure: %s", "unsupported upgrade attempt");
+
+	enum http_errno err = HTTP_PARSER_ERRNO (&self->parser);
+	if (err == HPE_PAUSED)
+	{
+		self->pending_fake_starter = true;
+		client_co_process (self);
+	}
+	else if (err != HPE_OK)
+		exit_fatal ("protocol failure: %s", http_errno_description (err));
+}
+
+static void
+client_co_on_data (struct co_context *self, const char *data, size_t len)
+{
+	size_t n_parsed = 0;
+	do
+	{
+		client_co_parse (self, data, len, &n_parsed);
+		data += n_parsed;
+	}
+	while ((len -= n_parsed));
+}
+
+static void
+client_co_run (struct server_context *ctx)
+{
+	struct co_context self = {};
+	self.ctx = ctx;
+	self.message = str_make ();
+	http_parser_init (&self.parser, HTTP_REQUEST);
+	self.parser.data = &self;
+	self.pending_fake_starter = true;
+
+	hard_assert (set_blocking (STDIN_FILENO, false));
+	struct str buf = str_make ();
+	struct pollfd pfd = { .fd = STDIN_FILENO, .events = POLLIN };
+	while (true)
+	{
+		if (poll (&pfd, 1, -1) <= 0)
+			exit_fatal ("poll: %s", strerror (errno));
+
+		str_remove_slice (&buf, 0, buf.len);
+		enum socket_io_result result = socket_io_try_read (pfd.fd, &buf);
+		int errno_saved = errno;
+
+		if (buf.len)
+			client_co_on_data (&self, buf.str, buf.len);
+		if (result == SOCKET_IO_ERROR)
+			exit_fatal ("read: %s", strerror (errno_saved));
+		if (result == SOCKET_IO_EOF)
+			break;
+	}
+	str_free (&buf);
+	str_free (&self.message);
+}
+
 // --- Basic server stuff ------------------------------------------------------
 
 typedef struct client *(*client_create_fn) (EV_P_ int sock_fd);
@@ -2911,11 +3115,12 @@ daemonize (struct server_context *ctx)
 }
 
 static void
-parse_program_arguments (int argc, char **argv)
+parse_program_arguments (int argc, char **argv, bool *running_as_slave)
 {
 	static const struct opt opts[] =
 	{
 		{ 't', "test", NULL, 0, "self-test" },
+		{ 's', "slave", NULL, 0, "co-process mode" },
 		{ 'd', "debug", NULL, 0, "run in debug mode" },
 		{ 'h', "help", NULL, 0, "display this help and exit" },
 		{ 'V', "version", NULL, 0, "output version information and exit" },
@@ -2935,6 +3140,9 @@ parse_program_arguments (int argc, char **argv)
 	case 't':
 		test_main (argc, argv);
 		exit (EXIT_SUCCESS);
+	case 's':
+		*running_as_slave = true;
+		break;
 	case 'd':
 		g_debug_mode = true;
 		break;
@@ -2967,7 +3175,8 @@ parse_program_arguments (int argc, char **argv)
 int
 main (int argc, char *argv[])
 {
-	parse_program_arguments (argc, argv);
+	bool running_as_a_slave = false;
+	parse_program_arguments (argc, argv, &running_as_a_slave);
 
 	print_status (PROGRAM_NAME " " PROGRAM_VERSION " starting");
 
@@ -2982,6 +3191,15 @@ main (int argc, char *argv[])
 		exit (EXIT_FAILURE);
 	}
 
+	// There's a lot of unnecessary left-over scaffolding in this program,
+	// for testing purposes assume that everything is synchronous
+	if (running_as_a_slave)
+	{
+		client_co_run (&ctx);
+		server_context_free (&ctx);
+		return EXIT_SUCCESS;
+	}
+
 	struct ev_loop *loop;
 	if (!(loop = EV_DEFAULT))
 		exit_fatal ("libev initialization failed");