Build TLS as a loadable module

* Support BUILD_TLS=module to be loaded as a module via config file or
  command line. e.g. redis-server --loadmodule redis-tls.so
* Updates to redismodule.h to allow it to be used side by side with
  server.h by defining REDISMODULE_CORE_MODULE
* Changes to server.h, redismodule.h and module.c to avoid repeated
  type declarations (gcc 4.8 doesn't like these)
* Add a mechanism for non-ABI neutral modules (ones who include
  server.h) to refuse loading if they detect not being built together with
  redis (release.c)
* Fix wrong signature of RedisModuleDefragFunc, this could break
  compilation of a module, but not the ABI
* Move initialization of listeners in server.c to be after loading
  the modules
* Config TLS after initialization of listeners
* Init cluster after initialization of listeners
* Add TLS module to CI
* Fix a test suite race conditions:
  Now that the listeners are initialized later, it's not sufficient to
  wait for the PID message in the log, we need to wait for the "Server
  Initialized" message.
* Fix issues with moduleconfigs test as a result from start_server
  waiting for "Server Initialized"
* Fix issues with modules/infra test as a result of an additional module
  present

Notes about Sentinel:
Sentinel can't really rely on the tls module, since it uses hiredis to
initiate connections and depends on OpenSSL (won't be able to use any
other connection modules for that), so it was decided that when TLS is
built as a module, sentinel does not support TLS at all.
This means that it keeps using redis_tls_ctx and redis_tls_client_ctx directly.

Example code of config in redis-tls.so(may be use in the future):
RedisModuleString *tls_cfg = NULL;

void tlsInfo(RedisModuleInfoCtx *ctx, int for_crash_report) {
    UNUSED(for_crash_report);
    RedisModule_InfoAddSection(ctx, "");
    RedisModule_InfoAddFieldLongLong(ctx, "var", 42);
}

int tlsCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
{
    if (argc != 2) return RedisModule_WrongArity(ctx);
    return RedisModule_ReplyWithString(ctx, argv[1]);
}

RedisModuleString *getStringConfigCommand(const char *name, void *privdata) {
    REDISMODULE_NOT_USED(name);
    REDISMODULE_NOT_USED(privdata);
    return tls_cfg;
}

int setStringConfigCommand(const char *name, RedisModuleString *new, void *privdata, RedisModuleString **err) {
    REDISMODULE_NOT_USED(name);
    REDISMODULE_NOT_USED(err);
    REDISMODULE_NOT_USED(privdata);
    if (tls_cfg) RedisModule_FreeString(NULL, tls_cfg);
    RedisModule_RetainString(NULL, new);
    tls_cfg = new;
    return REDISMODULE_OK;
}

int RedisModule_OnLoad(void *ctx, RedisModuleString **argv, int argc)
{
    ....
    if (RedisModule_CreateCommand(ctx,"tls",tlsCommand,"",0,0,0) == REDISMODULE_ERR)
        return REDISMODULE_ERR;

    if (RedisModule_RegisterStringConfig(ctx, "cfg", "", REDISMODULE_CONFIG_DEFAULT, getStringConfigCommand, setStringConfigCommand, NULL, NULL) == REDISMODULE_ERR)
        return REDISMODULE_ERR;

    if (RedisModule_LoadConfigs(ctx) == REDISMODULE_ERR) {
        if (tls_cfg) {
            RedisModule_FreeString(ctx, tls_cfg);
            tls_cfg = NULL;
        }
        return REDISMODULE_ERR;
    }
    ...
}

Co-authored-by: zhenwei pi <pizhenwei@bytedance.com>
Signed-off-by: zhenwei pi <pizhenwei@bytedance.com>

Author: oranagra

.github/workflows/ci.yml

@@ -24,7 +24,8 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: make
-        run: make SANITIZER=address REDIS_CFLAGS='-Werror'
+        # build with TLS module just for compilation coverage
+        run: make SANITIZER=address REDIS_CFLAGS='-Werror' BUILD_TLS=module
       - name: testprep
         run: sudo apt-get install tcl8.6 tclx -y
       - name: test

.github/workflows/daily.yml

@@ -539,27 +539,27 @@ jobs:
       run: |
         yum -y install centos-release-scl epel-release
         yum -y install devtoolset-7 openssl-devel openssl
-        scl enable devtoolset-7 "make BUILD_TLS=yes REDIS_CFLAGS='-Werror'"
+        scl enable devtoolset-7 "make BUILD_TLS=module REDIS_CFLAGS='-Werror'"
     - name: testprep
       run: |
         yum -y install tcl tcltls tclx
         ./utils/gen-test-certs.sh
     - name: test
       if: true && !contains(github.event.inputs.skiptests, 'redis')
       run: |
-        ./runtest --accurate --verbose --dump-logs --tls --dump-logs ${{github.event.inputs.test_args}}
+        ./runtest --accurate --verbose --dump-logs --tls-module --dump-logs ${{github.event.inputs.test_args}}
     - name: module api test
       if: true && !contains(github.event.inputs.skiptests, 'modules')
       run: |
-        ./runtest-moduleapi --verbose --dump-logs --tls --dump-logs ${{github.event.inputs.test_args}}
+        ./runtest-moduleapi --verbose --dump-logs --tls-module --dump-logs ${{github.event.inputs.test_args}}
     - name: sentinel tests
       if: true && !contains(github.event.inputs.skiptests, 'sentinel')
       run: |
-        ./runtest-sentinel --tls ${{github.event.inputs.cluster_test_args}}
+        ./runtest-sentinel ${{github.event.inputs.cluster_test_args}}
     - name: cluster tests
       if: true && !contains(github.event.inputs.skiptests, 'cluster')
       run: |
-        ./runtest-cluster --tls ${{github.event.inputs.cluster_test_args}}
+        ./runtest-cluster --tls-module ${{github.event.inputs.cluster_test_args}}
 
   test-centos7-tls-no-tls:
     runs-on: ubuntu-latest
@@ -582,7 +582,7 @@ jobs:
       run: |
         yum -y install centos-release-scl epel-release
         yum -y install devtoolset-7 openssl-devel openssl
-        scl enable devtoolset-7 "make BUILD_TLS=yes REDIS_CFLAGS='-Werror'"
+        scl enable devtoolset-7 "make BUILD_TLS=module REDIS_CFLAGS='-Werror'"
     - name: testprep
       run: |
         yum -y install tcl tcltls tclx

TLS.md

@@ -9,8 +9,14 @@ Getting Started
 To build with TLS support you'll need OpenSSL development libraries (e.g.
 libssl-dev on Debian/Ubuntu).
 
+To build TLS support as Redis built-in:
 Run `make BUILD_TLS=yes`.
 
+Or to build TLS as Redis module:
+Run `make BUILD_TLS=module`.
+
+Note that sentinel mode does not support TLS module.
+
 ### Tests
 
 To run Redis test suite with TLS, you'll need TLS support for TCL (i.e.
@@ -22,16 +28,27 @@ To run Redis test suite with TLS, you'll need TLS support for TCL (i.e.
 2. Run `./runtest --tls` or `./runtest-cluster --tls` to run Redis and Redis
    Cluster tests in TLS mode.
 
+3. Run `./runtest --tls-module` or `./runtest-cluster --tls-module` to
+   run Redis and Redis cluster tests in TLS mode with Redis module.
+
 ### Running manually
 
 To manually run a Redis server with TLS mode (assuming `gen-test-certs.sh` was
 invoked so sample certificates/keys are available):
 
+For TLS built-in mode:
     ./src/redis-server --tls-port 6379 --port 0 \
         --tls-cert-file ./tests/tls/redis.crt \
         --tls-key-file ./tests/tls/redis.key \
         --tls-ca-cert-file ./tests/tls/ca.crt
 
+For TLS module mode:
+    ./src/redis-server --tls-port 6379 --port 0 \
+        --tls-cert-file ./tests/tls/redis.crt \
+        --tls-key-file ./tests/tls/redis.key \
+        --tls-ca-cert-file ./tests/tls/ca.crt \
+        --loadmodule src/redis-tls.so
+
 To connect to this Redis server with `redis-cli`:
 
     ./src/redis-cli --tls \

deps/Makefile

@@ -44,7 +44,7 @@ distclean:
 
 .PHONY: distclean
 
-ifeq ($(BUILD_TLS),yes)
+ifneq (,$(filter $(BUILD_TLS),yes module))
     HIREDIS_MAKE_FLAGS = USE_SSL=1
 endif
 

src/Makefile

@@ -267,24 +267,41 @@ ifeq ($(MALLOC),jemalloc)
 	FINAL_LIBS := ../deps/jemalloc/lib/libjemalloc.a $(FINAL_LIBS)
 endif
 
-ifeq ($(BUILD_TLS),yes)
-	FINAL_CFLAGS+=-DUSE_OPENSSL $(OPENSSL_CFLAGS)
-	FINAL_LDFLAGS+=$(OPENSSL_LDFLAGS)
-	LIBSSL_PKGCONFIG := $(shell $(PKG_CONFIG) --exists libssl && echo $$?)
+# LIBSSL & LIBCRYPTO
+LIBSSL_LIBS=
+LIBSSL_PKGCONFIG := $(shell $(PKG_CONFIG) --exists libssl && echo $$?)
 ifeq ($(LIBSSL_PKGCONFIG),0)
 	LIBSSL_LIBS=$(shell $(PKG_CONFIG) --libs libssl)
 else
 	LIBSSL_LIBS=-lssl
 endif
-	LIBCRYPTO_PKGCONFIG := $(shell $(PKG_CONFIG) --exists libcrypto && echo $$?)
+LIBCRYPTO_LIBS=
+LIBCRYPTO_PKGCONFIG := $(shell $(PKG_CONFIG) --exists libcrypto && echo $$?)
 ifeq ($(LIBCRYPTO_PKGCONFIG),0)
 	LIBCRYPTO_LIBS=$(shell $(PKG_CONFIG) --libs libcrypto)
 else
 	LIBCRYPTO_LIBS=-lcrypto
 endif
+
+BUILD_NO:=0
+BUILD_YES:=1
+BUILD_MODULE:=2
+ifeq ($(BUILD_TLS),yes)
+	FINAL_CFLAGS+=-DUSE_OPENSSL=$(BUILD_YES) $(OPENSSL_CFLAGS) -DBUILD_TLS_MODULE=$(BUILD_NO)
+	FINAL_LDFLAGS+=$(OPENSSL_LDFLAGS)
 	FINAL_LIBS += ../deps/hiredis/libhiredis_ssl.a $(LIBSSL_LIBS) $(LIBCRYPTO_LIBS)
 endif
 
+TLS_MODULE=
+TLS_MODULE_NAME:=redis-tls$(PROG_SUFFIX).so
+TLS_MODULE_CFLAGS:=$(FINAL_CFLAGS)
+ifeq ($(BUILD_TLS),module)
+	FINAL_CFLAGS+=-DUSE_OPENSSL=$(BUILD_MODULE) $(OPENSSL_CFLAGS)
+	TLS_CLIENT_LIBS = ../deps/hiredis/libhiredis_ssl.a $(LIBSSL_LIBS) $(LIBCRYPTO_LIBS)
+	TLS_MODULE=$(TLS_MODULE_NAME)
+	TLS_MODULE_CFLAGS+=-DUSE_OPENSSL=$(BUILD_MODULE) $(OPENSSL_CFLAGS) -DBUILD_TLS_MODULE=$(BUILD_MODULE)
+endif
+
 ifndef V
     define MAKE_INSTALL
         @printf '    %b %b\n' $(LINKCOLOR)INSTALL$(ENDCOLOR) $(BINCOLOR)$(1)$(ENDCOLOR) 1>&2
@@ -325,7 +342,7 @@ REDIS_CHECK_RDB_NAME=redis-check-rdb$(PROG_SUFFIX)
 REDIS_CHECK_AOF_NAME=redis-check-aof$(PROG_SUFFIX)
 ALL_SOURCES=$(sort $(patsubst %.o,%.c,$(REDIS_SERVER_OBJ) $(REDIS_CLI_OBJ) $(REDIS_BENCHMARK_OBJ)))
 
-all: $(REDIS_SERVER_NAME) $(REDIS_SENTINEL_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) $(REDIS_CHECK_RDB_NAME) $(REDIS_CHECK_AOF_NAME)
+all: $(REDIS_SERVER_NAME) $(REDIS_SENTINEL_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) $(REDIS_CHECK_RDB_NAME) $(REDIS_CHECK_AOF_NAME) $(TLS_MODULE)
 	@echo ""
 	@echo "Hint: It's a good idea to run 'make test' ;)"
 	@echo ""
@@ -385,13 +402,17 @@ $(REDIS_CHECK_RDB_NAME): $(REDIS_SERVER_NAME)
 $(REDIS_CHECK_AOF_NAME): $(REDIS_SERVER_NAME)
 	$(REDIS_INSTALL) $(REDIS_SERVER_NAME) $(REDIS_CHECK_AOF_NAME)
 
+# redis-tls.so
+$(TLS_MODULE_NAME): $(REDIS_SERVER_NAME)
+	$(QUIET_CC)$(CC) -o $@ tls.c -shared -fPIC $(TLS_MODULE_CFLAGS) $(TLS_CLIENT_LIBS)
+
 # redis-cli
 $(REDIS_CLI_NAME): $(REDIS_CLI_OBJ)
-	$(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/linenoise/linenoise.o $(FINAL_LIBS)
+	$(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/linenoise/linenoise.o $(FINAL_LIBS) $(TLS_CLIENT_LIBS)
 
 # redis-benchmark
 $(REDIS_BENCHMARK_NAME): $(REDIS_BENCHMARK_OBJ)
-	$(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/hdr_histogram/libhdrhistogram.a $(FINAL_LIBS)
+	$(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/hdr_histogram/libhdrhistogram.a $(FINAL_LIBS) $(TLS_CLIENT_LIBS)
 
 DEP = $(REDIS_SERVER_OBJ:%.o=%.d) $(REDIS_CLI_OBJ:%.o=%.d) $(REDIS_BENCHMARK_OBJ:%.o=%.d)
 -include $(DEP)
@@ -410,7 +431,7 @@ commands.c: commands/*.json ../utils/generate-command-code.py
 endif
 
 clean:
-	rm -rf $(REDIS_SERVER_NAME) $(REDIS_SENTINEL_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) $(REDIS_CHECK_RDB_NAME) $(REDIS_CHECK_AOF_NAME) *.o *.gcda *.gcno *.gcov redis.info lcov-html Makefile.dep
+	rm -rf $(REDIS_SERVER_NAME) $(REDIS_SENTINEL_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) $(REDIS_CHECK_RDB_NAME) $(REDIS_CHECK_AOF_NAME) *.o *.gcda *.gcno *.gcov redis.info lcov-html Makefile.dep *.so
 	rm -f $(DEP)
 
 .PHONY: clean

src/connection.h

@@ -107,8 +107,6 @@ typedef struct ConnectionType {
 
     /* TLS specified methods */
     sds (*get_peer_cert)(struct connection *conn);
-    void* (*get_ctx)(void);
-    void* (*get_client_ctx)(void);
 } ConnectionType;
 
 struct connection {
@@ -355,15 +353,6 @@ int connKeepAlive(connection *conn, int interval);
 int connSendTimeout(connection *conn, long long ms);
 int connRecvTimeout(connection *conn, long long ms);
 
-/* Helpers for tls special considerations */
-static inline void *connTypeGetCtx(ConnectionType *ct) {
-    return ct->get_ctx();
-}
-
-static inline void *connTypeGetClientCtx(ConnectionType *ct) {
-    return ct->get_client_ctx();
-}
-
 /* Get cert for the secure connection */
 static inline sds connGetPeerCert(connection *conn) {
     if (conn->type->get_peer_cert) {
@@ -399,7 +388,7 @@ static inline connection *connCreate(ConnectionType *ct) {
     return ct->conn_create();
 }
 
-/* Create a accepted connection of specified type.
+/* Create an accepted connection of specified type.
  * priv is connection type specified argument */
 static inline connection *connCreateAccepted(ConnectionType *ct, int fd, void *priv) {
     return ct->conn_create_accepted(fd, priv);

src/mkreleasehdr.sh

@@ -11,4 +11,6 @@ test -f release.h || touch release.h
 echo "#define REDIS_GIT_SHA1 \"$GIT_SHA1\"" > release.h
 echo "#define REDIS_GIT_DIRTY \"$GIT_DIRTY\"" >> release.h
 echo "#define REDIS_BUILD_ID \"$BUILD_ID\"" >> release.h
+echo "#include \"version.h\"" >> release.h
+echo "#define REDIS_BUILD_ID_RAW REDIS_VERSION REDIS_BUILD_ID REDIS_GIT_DIRTY REDIS_GIT_SHA1" >> release.h
 touch release.c # Force recompile of release.c

src/module.c

@@ -69,14 +69,14 @@
  * pointers that have an API the module can call with them)
  * -------------------------------------------------------------------------- */
 
-typedef struct RedisModuleInfoCtx {
+struct RedisModuleInfoCtx {
     struct RedisModule *module;
     dict *requested_sections;
     sds info;           /* info string we collected so far */
     int sections;       /* number of sections we collected so far */
     int in_section;     /* indication if we're in an active section or not */
     int in_dict_field;  /* indication that we're currently appending to a dict */
-} RedisModuleInfoCtx;
+};
 
 /* This represents a shared API. Shared APIs will be used to populate
  * the server.sharedapi dictionary, mapping names of APIs exported by
@@ -12211,13 +12211,13 @@ const char *RM_GetCurrentCommandName(RedisModuleCtx *ctx) {
 /* The defrag context, used to manage state during calls to the data type
  * defrag callback.
  */
-typedef struct RedisModuleDefragCtx {
+struct RedisModuleDefragCtx {
     long defragged;
     long long int endtime;
     unsigned long *cursor;
     struct redisObject *key; /* Optional name of key processed, NULL when unknown. */
     int dbid;                /* The dbid of the key being processed, -1 when unknown. */
-} RedisModuleDefragCtx;
+};
 
 /* Register a defrag callback for global data, i.e. anything that the module
  * may allocate that is not tied to a specific data type.

src/redismodule.h

@@ -750,13 +750,41 @@ typedef enum {
     REDISMODULE_ACL_LOG_CHANNEL /* Channel authorization failure */
 } RedisModuleACLLogEntryReason;
 
+/* Incomplete structures needed by both the core and modules. */
+typedef struct RedisModuleString RedisModuleString;
+typedef struct RedisModuleIO RedisModuleIO;
+typedef struct RedisModuleDigest RedisModuleDigest;
+typedef struct RedisModuleInfoCtx RedisModuleInfoCtx;
+typedef struct RedisModuleDefragCtx RedisModuleDefragCtx;
+
+/* Function pointers needed by both the core and modules, these needs to be
+ * exposed since you can't cast a function pointer to (void *). */
+typedef void (*RedisModuleInfoFunc)(RedisModuleInfoCtx *ctx, int for_crash_report);
+typedef void (*RedisModuleDefragFunc)(RedisModuleDefragCtx *ctx);
+typedef void (*RedisModuleUserChangedFunc) (uint64_t client_id, void *privdata);
+
 /* ------------------------- End of common defines ------------------------ */
 
-#ifndef REDISMODULE_CORE
+#if defined REDISMODULE_CORE
+/* Things only defined for the modules core (server), not exported to modules
+ * that include this file. */
+
+#define RedisModuleString robj
+
+#endif /* defined REDISMODULE_CORE */
+
+#if !defined REDISMODULE_CORE && !defined REDISMODULE_CORE_MODULE
+/* Things defined for modules, but not for core-modules. */
 
 typedef long long mstime_t;
 typedef long long ustime_t;
 
+#endif /* !defined REDISMODULE_CORE && !defined REDISMODULE_CORE_MODULE */
+
+/* ----------- The rest of the defines are only for modules ----------------- */
+#if !defined REDISMODULE_CORE || defined REDISMODULE_CORE_MODULE
+/* Things defined for modules and core-modules. */
+
 /* Macro definitions specific to individual compilers */
 #ifndef REDISMODULE_ATTR_UNUSED
 #    ifdef __GNUC__
@@ -786,21 +814,16 @@ typedef long long ustime_t;
 typedef struct RedisModuleCtx RedisModuleCtx;
 typedef struct RedisModuleCommand RedisModuleCommand;
 typedef struct RedisModuleKey RedisModuleKey;
-typedef struct RedisModuleString RedisModuleString;
 typedef struct RedisModuleCallReply RedisModuleCallReply;
-typedef struct RedisModuleIO RedisModuleIO;
 typedef struct RedisModuleType RedisModuleType;
-typedef struct RedisModuleDigest RedisModuleDigest;
 typedef struct RedisModuleBlockedClient RedisModuleBlockedClient;
 typedef struct RedisModuleClusterInfo RedisModuleClusterInfo;
 typedef struct RedisModuleDict RedisModuleDict;
 typedef struct RedisModuleDictIter RedisModuleDictIter;
 typedef struct RedisModuleCommandFilterCtx RedisModuleCommandFilterCtx;
 typedef struct RedisModuleCommandFilter RedisModuleCommandFilter;
-typedef struct RedisModuleInfoCtx RedisModuleInfoCtx;
 typedef struct RedisModuleServerInfoData RedisModuleServerInfoData;
 typedef struct RedisModuleScanCursor RedisModuleScanCursor;
-typedef struct RedisModuleDefragCtx RedisModuleDefragCtx;
 typedef struct RedisModuleUser RedisModuleUser;
 typedef struct RedisModuleKeyOptCtx RedisModuleKeyOptCtx;
 
@@ -827,11 +850,8 @@ typedef void (*RedisModuleClusterMessageReceiver)(RedisModuleCtx *ctx, const cha
 typedef void (*RedisModuleTimerProc)(RedisModuleCtx *ctx, void *data);
 typedef void (*RedisModuleCommandFilterFunc) (RedisModuleCommandFilterCtx *filter);
 typedef void (*RedisModuleForkDoneHandler) (int exitcode, int bysignal, void *user_data);
-typedef void (*RedisModuleInfoFunc)(RedisModuleInfoCtx *ctx, int for_crash_report);
 typedef void (*RedisModuleScanCB)(RedisModuleCtx *ctx, RedisModuleString *keyname, RedisModuleKey *key, void *privdata);
 typedef void (*RedisModuleScanKeyCB)(RedisModuleKey *key, RedisModuleString *field, RedisModuleString *value, void *privdata);
-typedef void (*RedisModuleUserChangedFunc) (uint64_t client_id, void *privdata);
-typedef int (*RedisModuleDefragFunc)(RedisModuleDefragCtx *ctx);
 typedef RedisModuleString * (*RedisModuleConfigGetStringFunc)(const char *name, void *privdata);
 typedef long long (*RedisModuleConfigGetNumericFunc)(const char *name, void *privdata);
 typedef int (*RedisModuleConfigGetBoolFunc)(const char *name, void *privdata);
@@ -1559,11 +1579,5 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int
 
 #define RMAPI_FUNC_SUPPORTED(func) (func != NULL)
 
-#else
-
-/* Things only defined for the modules core, not exported to modules
- * including this file. */
-#define RedisModuleString robj
-
 #endif /* REDISMODULE_CORE */
 #endif /* REDISMODULE_H */

src/release.c

@@ -35,7 +35,6 @@
 #include <stdio.h>
 
 #include "release.h"
-#include "version.h"
 #include "crc64.h"
 
 char *redisGitSHA1(void) {
@@ -46,8 +45,12 @@ char *redisGitDirty(void) {
     return REDIS_GIT_DIRTY;
 }
 
+const char *redisBuildIdRaw(void) {
+    return REDIS_BUILD_ID_RAW;
+}
+
 uint64_t redisBuildId(void) {
-    char *buildid = REDIS_VERSION REDIS_BUILD_ID REDIS_GIT_DIRTY REDIS_GIT_SHA1;
+    char *buildid = REDIS_BUILD_ID_RAW;
 
     return crc64(0,(unsigned char*)buildid,strlen(buildid));
 }