diff --git a/README.md b/README.md index a50ef19..19da694 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,9 @@ This projects requires : - a syslog server (such as [syslog-ng](https://github.com/balabit/syslog-ng) or [rsyslog](https://github.com/rsyslog/rsyslog)) - the [lua-resty-logger-socket](https://github.com/cloudflare/lua-resty-logger-socket) module +This project has been tested with Apicast v3.2. It may work with newer or older version +but it may require some minor changes. + ## Installation If not already done, start your syslog server and configure it to listen @@ -83,7 +86,7 @@ Once, you get it to work on `apicast-staging`, you can do the same on `apicast-p The following section tries to evaluate the overhead of this module on apicast performances. -Performance tests have been run on a vanilla apicast and an apicast with this +Performance tests have been run on a vanilla apicast 3.0 and an apicast with this module. Both tests have been run with 1k requests and responses and 10k requests and responses. @@ -160,6 +163,40 @@ The requests and responses are serialized as follow: } ``` +## Configuration + +The following excerpt shows a sample configuration for this module, when used +as a service policy. + +```javascript +{ + "services":[ + { + "id":42, + "proxy":{ + "policy_chain":[ + { + "name":"custom.logger.verbose", // the verbose policy that lays in the ./gateway/custom/logger/ directory + "configuration": { + "syslog_host": "syslog.acme.test", // the hostname of the syslog server + "syslog_port": 1601, // the port of the syslog server + "syslog_protocol": "tcp", // the protocol to use to connect to the syslog server (tcp or udp) + "syslog_flush_limit": "0", // the minimum number of bytes in the buffer before sending logs to the syslog server + "syslog_drop_limit": "1048576", // the maximum number of bytes in the buffer before starting to drop messages + "syslog_periodic_flush": "5", // the number of seconds between each log flush (0 to disable) + "payload_encoding": "base64" // the algorithm used to encode the payload ('base64' or 'none') + } + }, + { + "name":"apicast.policy.apicast" // also keep the default apicast behavior + } + ] + } + } + ] +} +``` + ## Development First of all, setup your development environment as explained [here](https://github.com/3scale/apicast/tree/master#development--testing). @@ -168,41 +205,26 @@ Then, issue the following commands: ``` git clone https://github.com/nmasse-itix/apicast-logger.git git clone https://github.com/3scale/apicast.git +git clone https://github.com/cloudflare/lua-resty-logger-socket.git +export GIT_ROOT=$PWD cd apicast luarocks make apicast/*.rockspec --local -ln -s ../apicast-logger custom +mkdir gateway/src/custom +ln -s $GIT_ROOT/apicast-logger/ gateway/src/custom/logger/ +cd gateway/src/resty +ln -s $GIT_ROOT/lua-resty-logger-socket/lib/resty/logger/ logger +cd - ``` -Configure your apicast as explained [here](https://github.com/3scale/apicast/blob/master/doc/parameters.md). +Configure your apicast with a local configuration: ``` -export THREESCALE_DEPLOYMENT_ENV=sandbox -export THREESCALE_PORTAL_ENDPOINT=https://@-admin.3scale.net +export THREESCALE_CONFIG_FILE=$GIT_ROOT/apicast-logger/config.json export APICAST_LOG_LEVEL=debug ``` -Configure the module: -``` -export SYSLOG_HOST=127.0.0.1.xip.io -export SYSLOG_PORT=1601 -export SYSLOG_PROTOCOL=tcp -export APICAST_MODULE=custom/verbose -``` - -Plain text logging of payload without base64 encoding: -``` -export APICAST_PAYLOAD_BASE64=false -``` - -Then, you need to register a resolver in the nginx configuration (example using the Google DNS): -``` -cat < apicast/apicast.d/resolver.conf -resolver 8.8.8.8 -EOF -``` - Finally, launch apicast: ``` -bin/apicast -i 0 -m off +bin/apicast --dev ``` And in another terminal, launch netcat so that you can simulate a syslog server: @@ -210,6 +232,11 @@ And in another terminal, launch netcat so that you can simulate a syslog server: nc -l 1601 ``` +## References + +The following reading is recommended if you plan to develop on this module: + - [How to develop policies for Apicast](https://github.com/3scale/apicast/blob/master/doc/policies.md) + ## Troubleshooting When troubleshooting, keep in mind that the underlying `lua-resty-logger-socket` diff --git a/config.json b/config.json index 5f8ec88..07c78b1 100644 --- a/config.json +++ b/config.json @@ -18,14 +18,52 @@ }, "auth_user_key": "user_key", "credentials_location": "query", + "policy_chain": [ + { + "name": "apicast.policy.apicast" + }, + { + "name": "apicast.policy.phase_logger" + }, + { + "name": "custom.logger.verbose", + "configuration": { + "syslog_host": "127.0.0.1.xip.io", + "syslog_port": "1601", + "syslog_protocol": "tcp", + "payload_encoding": "base64" + } + } + ], "proxy_rules": [ { + "http_method": "GET", + "pattern": "/", + "metric_system_name": "hits", + "delta": 1, + "parameters": [], + "querystring_parameters": {} + },{ "http_method": "POST", "pattern": "/", "metric_system_name": "hits", "delta": 1, "parameters": [], "querystring_parameters": {} + },{ + "http_method": "PUT", + "pattern": "/", + "metric_system_name": "hits", + "delta": 1, + "parameters": [], + "querystring_parameters": {} + },{ + "http_method": "DELETE", + "pattern": "/", + "metric_system_name": "hits", + "delta": 1, + "parameters": [], + "querystring_parameters": {} } ] } diff --git a/resolver.conf b/resolver.conf deleted file mode 100644 index d81b8eb..0000000 --- a/resolver.conf +++ /dev/null @@ -1,11 +0,0 @@ -# Nginx Configuration - To be placed in the 'apicast.d' folder. -# -# We need to add a resolver stanza since the underlying 'lua-resty-logger-socket' -# module use the nginx resolver and not ours. -# -# See https://github.com/openresty/lua-nginx-module#tcpsockconnect -# -# On Apicast Docker images, a built-in dnsmasq resolver is embedded -# and available on localhost port 5353 -# -resolver 127.0.0.1:5353; diff --git a/verbose.lua b/verbose.lua index 70355e2..72b034f 100644 --- a/verbose.lua +++ b/verbose.lua @@ -1,73 +1,68 @@ -local apicast = require('apicast').new() +local policy = require('apicast.policy') +local _M = policy.new('Verbose Logger Policy') + local cjson = require('cjson') local logger = require("resty.logger.socket") -local _M = { _VERSION = '0.0' } -local mt = { __index = setmetatable(_M, { __index = apicast }) } - -function _M.new() - return setmetatable({}, mt) -end - -local host -local port -local proto -local flush_limit -local drop_limit - -- Parse and validate the parameters: --- SYSLOG_HOST => the hostname of the syslog server --- SYSLOG_PORT => the port of the syslog server --- SYSLOG_PROTOCOL => the protocol to use to connect to the syslog server (tcp or udp) --- SYSLOG_FLUSH_LIMIT => the minimum number of bytes in the buffer before sending logs to the syslog server --- SYSLOG_DROP_LIMIT => the maximum number of bytes in the buffer before starting to drop messages --- SYSLOG_PERIODIC_FLUSH => the number of seconds between each log flush (0 to disable) +-- syslog_host => the hostname of the syslog server +-- syslog_port => the port of the syslog server +-- syslog_protocol => the protocol to use to connect to the syslog server (tcp or udp) +-- syslog_flush_limit => the minimum number of bytes in the buffer before sending logs to the syslog server +-- syslog_drop_limit => the maximum number of bytes in the buffer before starting to drop messages +-- syslog_periodic_flush => the number of seconds between each log flush (0 to disable) +-- payload_encoding => the algorithm used to encode the payload ('base64' or 'none') -- -function _M:init() - host = os.getenv('SYSLOG_HOST') - port = os.getenv('SYSLOG_PORT') - proto = os.getenv('SYSLOG_PROTOCOL') or 'tcp' - base64_flag = os.getenv('APICAST_PAYLOAD_BASE64') or 'true' - flush_limit = os.getenv('SYSLOG_FLUSH_LIMIT') or '0' - periodic_flush = os.getenv('SYSLOG_PERIODIC_FLUSH') or '5' - drop_limit = os.getenv('SYSLOG_DROP_LIMIT') or '1048576' - - if (host == nil or host == "") then - ngx.log(ngx.ERR, "The environment SYSLOG_HOST is NOT defined !") +local new = _M.new +function _M.new(config) + local self = new() + + -- Optional parameters + self.proto = config.syslog_protocol or 'tcp' + self.base64_flag = config.payload_encoding and (config.payload_encoding == 'base64') + self.flush_limit = config.syslog_flush_limit or 0 + self.periodic_flush = config.syslog_periodic_flush or 5 + self.drop_limit = config.syslog_drop_limit or 1048576 + + -- Required parameters + if (config.syslog_host == nil or config.syslog_host == "") then + ngx.log(ngx.ERR, "The configuration option syslog_host is NOT defined !") end - if (port == nil or port == "") then - ngx.log(ngx.ERR, "The environment SYSLOG_PORT is NOT defined !") + if (config.syslog_port == nil or config.syslog_port == "") then + ngx.log(ngx.ERR, "The configuration option syslog_port is NOT defined !") end - port = tonumber(port) - flush_limit = tonumber(flush_limit) - drop_limit = tonumber(drop_limit) - periodic_flush = tonumber(periodic_flush) + self.host = config.syslog_host + self.port = tonumber(config.syslog_port) - ngx.log(ngx.WARN, "Sending custom logs to " .. proto .. "://" .. (host or "") .. ":" .. (port or "") .. " with flush_limit = " .. flush_limit .. " bytes, periodic_flush = " .. periodic_flush .. " sec. and drop_limit = " .. drop_limit .. " bytes") + ngx.log(ngx.WARN, "Sending custom logs to " .. self.proto .. "://" .. (self.host or "") .. ":" .. (self.port or "") .. " with flush_limit = " .. self.flush_limit .. " bytes, periodic_flush = " .. self.periodic_flush .. " sec. and drop_limit = " .. self.drop_limit .. " bytes") - return apicast:init() + return self end -- Initialize the underlying logging module. Since the module calls 'timer_at' -- during initialization, we need to call it from a init_worker_by_lua block. -- function _M:init_worker() - ngx.log(ngx.INFO, "Initializing the underlying logger") + ensure_logger_is_initted(self) +end + +local function ensure_logger_is_initted(self) if not logger.initted() then + ngx.log(ngx.INFO, "Initializing the underlying logger") -- default parameters local params = { - host = host, - port = port, - sock_type = proto, - flush_limit = flush_limit, - drop_limit = drop_limit + host = self.host, + port = self.port, + sock_type = self.proto, + flush_limit = self.flush_limit, + drop_limit = self.drop_limit } -- periodic_flush == 0 means 'disable this feature' - if periodic_flush > 0 then - params["periodic_flush"] = periodic_flush + if self.periodic_flush > 0 then + params["periodic_flush"] = self.periodic_flush end -- initialize the logger @@ -76,12 +71,9 @@ function _M:init_worker() ngx.log(ngx.ERR, "failed to initialize the logger: ", err) end end - - return apicast:init_worker() end - -function do_log(payload) +local function do_log(payload) -- construct the custom access log message in -- the Lua variable "msg" -- @@ -95,8 +87,8 @@ end -- This function is called for each chunk of response received from upstream server -- when the last chunk is received, ngx.arg[2] is true. -function _M.body_filter() - ngx.ctx.buffered = (ngx.ctx.buffered or "") .. ngx.arg[1] +function _M:body_filter(context) + context.buffered = (context.buffered or "") .. ngx.arg[1] if ngx.arg[2] then -- EOF local dict = {} @@ -104,7 +96,7 @@ function _M.body_filter() -- Gather information of the request local request = {} if ngx.var.request_body then - if (base64_flag == 'true') then + if (self.base64_flag) then request["body"] = ngx.encode_base64(ngx.var.request_body) else request["body"] = ngx.var.request_body @@ -126,11 +118,11 @@ function _M.body_filter() -- Gather information of the response local response = {} - if ngx.ctx.buffered then - if (base64_flag == 'true') then - response["body"] = ngx.encode_base64(ngx.ctx.buffered) + if context.buffered then + if (self.base64_flag) then + response["body"] = ngx.encode_base64(context.buffered) else - response["body"] = ngx.ctx.buffered + response["body"] = context.buffered end end response["headers"] = ngx.resp.get_headers() @@ -149,9 +141,9 @@ function _M.body_filter() upstream["status"] = ngx.var.upstream_status dict["upstream"] = upstream + ensure_logger_is_initted(self) do_log(cjson.encode(dict)) end - return apicast:body_filter() end return _M