improve doc and improve reliability

README.md

@@ -31,12 +31,13 @@ oc create configmap syslog-ng --from-file=syslog-ng.conf
 oc volume dc/syslog-ng --add --name=conf --mount-path /etc/syslog-ng/conf.d/ --type=configmap --configmap-name=syslog-ng
 ```
 
-Then, update your `apicast-{staging,production}` to embed the required code, module and environment variables.
+Then, update your `apicast-staging` to embed the required code,
+module and environment variables as explained:
 
 Put `resolver.conf` in `/opt/app-root/src/apicast.d/resolver.conf`:
 ```
-oc create configmap resolver --from-file=resolver.conf
+oc create configmap apicast.d --from-file=resolver.conf
-oc volume dc/apicast-staging --add --name=resolver --mount-path /opt/app-root/src/apicast.d/ --type=configmap --configmap-name=resolver
+oc volume dc/apicast-staging --add --name=apicastd --mount-path /opt/app-root/src/apicast.d/ --type=configmap --configmap-name=apicast.d
 ```
 
 Put the `lua-resty-logger-socket` module in `/opt/app-root/src/src/resty/logger/`:
@@ -48,8 +49,8 @@ oc volume dc/apicast-staging --add --name=lua-resty-logger-socket --mount-path /
 
 Put the `verbose.lua` module in `/opt/app-root/src/src/custom/`:
 ```
-oc create configmap apicast-logging --from-file=verbose.lua
+oc create configmap apicast-custom-module --from-file=verbose.lua
-oc volume dc/apicast-staging --add --name=apicast-logging --mount-path /opt/app-root/src/src/custom/ --type=configmap --configmap-name=apicast-logging
+oc volume dc/apicast-staging --add --name=apicast-custom-module --mount-path /opt/app-root/src/src/custom/ --type=configmap --configmap-name=apicast-custom-module
 ```
 
 Set the configuration required by `verbose.lua` as environment variables and re-deploy apicast:
@@ -57,10 +58,26 @@ Set the configuration required by `verbose.lua` as environment variables and re-
 oc env dc/apicast-staging APICAST_MODULE=custom/verbose
 oc env dc/apicast-staging SYSLOG_PROTOCOL=tcp
 oc env dc/apicast-staging SYSLOG_PORT=601
-oc env dc/apicast-staging SYSLOG_HOST=syslog-ng
+oc env dc/apicast-staging SYSLOG_HOST=syslog-ng.3scale.svc.cluster.local
 oc rollout latest apicast-staging
 ```
 
+**NOTE:** You need to adjust the value of `SYSLOG_HOST` to match your environment.
+Namely, make sure you are using a FQDN that resolves to your syslog server.
+If the syslog server is deployed in OpenShift, it needs to be in the same project
+as the apicast (of course, unless you are using a flat network...).
+
+In an OpenShift environment, the `SYSLOG_HOST` would look like:
+```
+<service-name>.<project>.svc.cluster.local
+```
+
+**WARNING:** You cannot use a short name (ie `syslog-ng`). It has to be a FQDN.
+This is because nginx does not rely on the standard glibc API `gethostbyname` but
+uses instead a custom resolver.
+
+Once, you get it to work on `apicast-staging`, you can do the same on `apicast-production`.
+
 ## Message format
 
 The requests and responses are serialized as follow:
@@ -110,23 +127,44 @@ The requests and responses are serialized as follow:
 
 First of all, setup your development environment as explained [here](https://github.com/3scale/apicast/tree/master#development--testing).
 
-Then, issue the following commands :
+Then, issue the following commands:
 ```
-git clone TODO
+git clone https://github.com/nmasse-itix/apicast-logger.git
 git clone https://github.com/3scale/apicast.git
 cd apicast
-git checkout -b 3.0-stable
 luarocks make apicast/*.rockspec --local
+ln -s ../apicast-logger custom
+```
+
+Configure your apicast as explained [here](https://github.com/3scale/apicast/blob/master/doc/parameters.md).
+```
 export THREESCALE_DEPLOYMENT_ENV=sandbox
 export THREESCALE_PORTAL_ENDPOINT=https://<YOUR-TOKEN-HERE>@<YOUR-TENANT-HERE>-admin.3scale.net
-export SYSLOG_HOST=localhost
+export APICAST_LOG_LEVEL=debug
-export SYSLOG_PORT=601
+```
-export SYSLOG_PROTOCOL=tcp
 
-ln -s ../apicast-logger custom
+Configure the module:
+```
+export SYSLOG_HOST=127.0.0.1.xip.io
+export SYSLOG_PORT=1601
+export SYSLOG_PROTOCOL=tcp
 export APICAST_MODULE=custom/verbose
+```
 
-bin/apicast -vvvv -i 0 -m off
+Then, you need to register a resolver in the nginx configuration (example using the Google DNS):
+```
+cat <<EOF > apicast/apicast.d/resolver.conf
+resolver 8.8.8.8:
+```
+
+Finally, launch apicast:
+```
+bin/apicast -i 0 -m off
+```
+
+And in another terminal, launch netcat so that you can simulate a syslog server:
+```
+nc -l 1601
 ```
 
 ## Troubleshooting

verbose.lua

@@ -9,12 +9,27 @@ function _M.new()
   return setmetatable({}, mt)
 end
 
+local host
+local port
+local proto
+local flush_limit
+local drop_limit
+
+-- Parse and validate the parameters:
+--   SYSLOG_HOST => the hostname of the syslog server
+--   SYSLOG_PORT => the port of the syslog server
+--   SYSLOG_PROTOCOL => the protocol to use to connect to the syslog server (tcp or udp)
+--   SYSLOG_FLUSH_LIMIT => the minimum number of bytes in the buffer before sending logs to the syslog server
+--   SYSLOG_DROP_LIMIT => the maximum number of bytes in the buffer before starting to drop messages
+--   SYSLOG_PERIODIC_FLUSH => the number of seconds between each log flush (0 to disable)
+--
 function _M:init()
-  local host = os.getenv('SYSLOG_HOST')
+  host = os.getenv('SYSLOG_HOST')
-  local port = os.getenv('SYSLOG_PORT')
+  port = os.getenv('SYSLOG_PORT')
-  local proto = os.getenv('SYSLOG_PROTOCOL') or 'tcp'
+  proto = os.getenv('SYSLOG_PROTOCOL') or 'tcp'
-  local flush_limit = os.getenv('SYSLOG_FLUSH_LIMIT') or '0'
+  flush_limit = os.getenv('SYSLOG_FLUSH_LIMIT') or '0'
-  local drop_limit = os.getenv('SYSLOG_DROP_LIMIT') or '1048576'
+  periodic_flush = os.getenv('SYSLOG_PERIODIC_FLUSH') or '5'
+  drop_limit = os.getenv('SYSLOG_DROP_LIMIT') or '1048576'
 
   if (host == nil or host == "") then
     ngx.log(ngx.ERR, "The environment SYSLOG_HOST is NOT defined !")
@@ -27,8 +42,18 @@ function _M:init()
   port = tonumber(port)
   flush_limit = tonumber(flush_limit)
   drop_limit = tonumber(drop_limit)
-  ngx.log(ngx.WARN, "Sending custom logs to " .. proto .. "://" .. host .. ":" .. port .. " with flush_limit = " .. flush_limit .. " and drop_limit = " .. drop_limit)
+  periodic_flush = tonumber(periodic_flush)
 
+  ngx.log(ngx.WARN, "Sending custom logs to " .. proto .. "://" .. (host or "") .. ":" .. (port or "") .. " with flush_limit = " .. flush_limit .. " bytes, periodic_flush = " .. periodic_flush .. " sec. and drop_limit = " .. drop_limit .. " bytes")
+
+  return apicast:init()
+end
+
+-- Initialize the underlying logging module. Since the module calls 'timer_at'
+-- during initialization, we need to call it from a init_worker_by_lua block.
+--
+function _M:init_worker()
+  ngx.log(ngx.INFO, "Initializing the underlying logger")
   if not logger.initted() then
       local ok, err = logger.init{
           host = host,
@@ -36,20 +61,24 @@ function _M:init()
           sock_type = proto,
           flush_limit = flush_limit,
           drop_limit = drop_limit,
+          periodic_flush = periodic_flush
       }
       if not ok then
           ngx.log(ngx.ERR, "failed to initialize the logger: ", err)
       end
   end
 
-  return apicast:init()
+  return apicast:init_worker()
 end
 
 
 function do_log(payload)
   -- construct the custom access log message in
   -- the Lua variable "msg"
-  local bytes, err = logger.log(payload)
+  --
+  -- do not forget the \n in order to have one request per line on the syslog server
+  -- 
+  local bytes, err = logger.log(payload .. "\n")
   if err then
       ngx.log(ngx.ERR, "failed to log message: ", err)
   end