upgrade openssl to 1.1.1i

.github/workflows/build.yml

@@ -0,0 +1,17 @@
+name: build
+
+on:
+  push:
+
+jobs:
+  build:
+    runs-on: ${{ matrix.builder }}
+    strategy:
+      matrix:
+        builder: [macos-latest, ubuntu-latest]
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: build
+        run: make

.gitignore

@@ -1,2 +1,2 @@
-obj/*
+obj/
 wrk

CHANGES

@@ -0,0 +1,24 @@
+master
+
+ * Require OpenSSL 1.1.0+
+
+wrk 4.0.2
+
+ * Send hostname using TLS SNI.
+ * Add optional WITH_OPENSSL and WITH_LUAJIT to use system libs.
+ * Bundle OpenSSL 1.0.2.
+ * delay() can return milliseconds to delay sending next request.
+
+wrk 4.0.0
+
+ * The wrk global variable is the only global defined by default.
+ * wrk.init() calls the global init(), remove calls to wrk.init().
+ * Add wrk.lookup(host, port) and wrk.connect(addr) functions.
+ * Add setup phase that calls the global setup() for each thread.
+ * Allow assignment to thread.addr to specify the server address.
+ * Add thread:set(name, value), thread:get(name), and thread:stop().
+ * Record latency for every request instead of random samples.
+ * Latency and requests in done() are now callable, not indexable.
+ * Only record timeouts when a response is actually received.
+ * Remove calibration phase and record rate at fixed interval.
+ * Improve correction of coordinated omission.

INSTALL

@@ -0,0 +1,29 @@
+Overview
+
+  wrk should build on most UNIX-like operating systems and
+  architectures that have GNU make and are supported by LuaJIT and
+  OpenSSL. Some systems may require additional CFLAGS or LDFLAGS,
+  see the top of the Makefile for examples
+
+  In many cases simply running `make` (often `gmake` on *BSD) will
+  do the trick.
+
+Dependencies
+
+  wrk requires LuaJIT and OpenSSL and is distributed with appropriate
+  versions that will be unpacked and built as necessary.
+
+  If you are building wrk packages for an OS distribution or otherwise
+  prefer to use system versions of dependencies you may specify their
+  location when invoking make with one or more of:
+
+    WITH_LUAJIT
+    WITH_OPENSSL
+
+  For example to use the system version of both libraries on Linux:
+
+    make WITH_LUAJIT=/usr WITH_OPENSSL=/usr
+
+  Or to use the Homebrew version of OpenSSL on Mac OS X:
+
+    make WITH_OPENSSL=/usr/local/opt/openssl

LICENSE

@@ -1,7 +1,6 @@
 
-                                 Apache License
+                          Modified Apache 2.0 License
-                           Version 2.0, January 2004
+                         Version 2.0.1, February 2015
-                        http://www.apache.org/licenses/
 
    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
 
@@ -121,6 +120,12 @@
           that such additional attribution notices cannot be construed
           as modifying the License.
 
+      (e) If the Derivative Work includes substantial changes to features
+          or functionality of the Work, then you must remove the name of
+          the Work, and any derivation thereof, from all copies that you
+          distribute, whether in Source or Object form, except as required
+          in copyright, patent, trademark, and attribution notices.
+
       You may add Your own copyright statement to Your modifications and
       may provide additional or different license terms and conditions
       for use, reproduction, or distribution of Your modifications, or

Makefile

@@ -1,38 +1,110 @@
-CFLAGS  := -std=c99 -Wall -O2 -D_REENTRANT
+CFLAGS  += -std=c99 -Wall -O2 -D_REENTRANT
-LIBS    := -lpthread -lm
+LIBS    := -lm -lssl -lcrypto -lpthread
 
-TARGET  := $(shell uname -s | tr [A-Z] [a-z] 2>/dev/null || echo unknown)
+TARGET  := $(shell uname -s | tr '[A-Z]' '[a-z]' 2>/dev/null || echo unknown)
 
 ifeq ($(TARGET), sunos)
 	CFLAGS += -D_PTHREADS -D_POSIX_C_SOURCE=200112L
 	LIBS   += -lsocket
+else ifeq ($(TARGET), darwin)
+	export MACOSX_DEPLOYMENT_TARGET = $(shell sw_vers -productVersion)
+else ifeq ($(TARGET), linux)
+	CFLAGS  += -D_POSIX_C_SOURCE=200112L -D_BSD_SOURCE -D_DEFAULT_SOURCE
+	LIBS    += -ldl
+	LDFLAGS += -Wl,-E
+else ifeq ($(TARGET), freebsd)
+	CFLAGS  += -D_DECLARE_C99_LDBL_MATH
+	LDFLAGS += -Wl,-E
 endif
 
-SRC  := wrk.c aprintf.c stats.c units.c ae.c zmalloc.c http_parser.c tinymt64.c
+SRC  := wrk.c net.c ssl.c aprintf.c stats.c script.c units.c \
+		ae.c zmalloc.c http_parser.c
 BIN  := wrk
+VER  ?= $(shell git describe --tags --always --dirty)
 
 ODIR := obj
-OBJ  := $(patsubst %.c,$(ODIR)/%.o,$(SRC))
+OBJ  := $(patsubst %.c,$(ODIR)/%.o,$(SRC)) $(ODIR)/bytecode.o $(ODIR)/version.o
+LIBS := -lluajit-5.1 $(LIBS)
+
+DEPS    :=
+CFLAGS  += -I$(ODIR)/include
+LDFLAGS += -L$(ODIR)/lib
+
+ifneq ($(WITH_LUAJIT),)
+	CFLAGS  += -I$(WITH_LUAJIT)/include
+	LDFLAGS += -L$(WITH_LUAJIT)/lib
+else
+	CFLAGS  += -I$(ODIR)/include/luajit-2.1
+	DEPS    += $(ODIR)/lib/libluajit-5.1.a
+endif
+
+ifneq ($(WITH_OPENSSL),)
+	CFLAGS  += -I$(WITH_OPENSSL)/include
+	LDFLAGS += -L$(WITH_OPENSSL)/lib
+else
+	DEPS += $(ODIR)/lib/libssl.a
+endif
 
 all: $(BIN)
 
 clean:
-	$(RM) $(BIN) obj/*
+	$(RM) -rf $(BIN) obj/*
 
 $(BIN): $(OBJ)
-	$(CC) $(LDFLAGS) -o $@ $^ $(LIBS)
+	@echo LINK $(BIN)
+	@$(CC) $(LDFLAGS) -o $@ $^ $(LIBS)
 
-$(OBJ): config.h Makefile | $(ODIR)
+$(OBJ): config.h Makefile $(DEPS) | $(ODIR)
 
 $(ODIR):
-	@mkdir $@
+	@mkdir -p $@
+
+$(ODIR)/bytecode.c: src/wrk.lua $(DEPS)
+	@echo LUAJIT $<
+	@$(SHELL) -c 'PATH="obj/bin:$(PATH)" luajit -b "$(CURDIR)/$<" "$(CURDIR)/$@"'
+
+$(ODIR)/version.o:
+	@echo 'const char *VERSION="$(VER)";' | $(CC) -xc -c -o $@ -
 
 $(ODIR)/%.o : %.c
-	$(CC) $(CFLAGS) -c -o $@ $<
+	@echo CC $<
+	@$(CC) $(CFLAGS) -c -o $@ $<
+
+# Dependencies
+
+LUAJIT  := $(notdir $(patsubst %.zip,%,$(wildcard deps/LuaJIT*.zip)))
+OPENSSL := $(notdir $(patsubst %.tar.gz,%,$(wildcard deps/openssl*.tar.gz)))
+
+OPENSSL_OPTS = no-shared no-psk no-srp no-dtls no-idea --prefix=$(abspath $(ODIR))
+
+$(ODIR)/$(LUAJIT): deps/$(LUAJIT).zip | $(ODIR)
+	echo $(LUAJIT)
+	@unzip -nd $(ODIR) $<
+
+$(ODIR)/$(OPENSSL): deps/$(OPENSSL).tar.gz | $(ODIR)
+	@tar -C $(ODIR) -xf $<
+
+$(ODIR)/lib/libluajit-5.1.a: $(ODIR)/$(LUAJIT)
+	@echo Building LuaJIT...
+	@$(MAKE) -C $< PREFIX=$(abspath $(ODIR)) BUILDMODE=static install
+	@cd $(ODIR)/bin && ln -s luajit-2.1.0-beta3 luajit
+
+$(ODIR)/lib/libssl.a: $(ODIR)/$(OPENSSL)
+	@echo Building OpenSSL...
+	@$(SHELL) -c "cd $< && ./config $(OPENSSL_OPTS)"
+	@$(MAKE) -C $< depend
+	@$(MAKE) -C $<
+	@$(MAKE) -C $< install_sw
+	@touch $@
+
+# ------------
 
 .PHONY: all clean
-.SUFFIXES:
+.PHONY: $(ODIR)/version.o
-.SUFFIXES: .c .o
 
-vpath %.c src
+.SUFFIXES:
-vpath %.h src
+.SUFFIXES: .c .o .lua
+
+vpath %.c   src
+vpath %.h   src
+vpath %.lua scripts

NOTICE

@@ -82,36 +82,27 @@ OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
 USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 =========================================================================
-==  Tiny Mersenne Twister (TinyMT) Notice                              ==
+==  LuaJIT Notice                                                      ==
 =========================================================================
 
-Copyright (c) 2011 Mutsuo Saito, Makoto Matsumoto, Hiroshima University
+LuaJIT -- a Just-In-Time Compiler for Lua. http://luajit.org/
-and The University of Tokyo. All rights reserved.
 
-Redistribution and use in source and binary forms, with or without
+Copyright (C) 2005-2013 Mike Pall. All rights reserved.
-modification, are permitted provided that the following conditions are
-met:
 
-    * Redistributions of source code must retain the above copyright
+Permission is hereby granted, free of charge, to any person obtaining a copy
-      notice, this list of conditions and the following disclaimer.
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
 
-    * Redistributions in binary form must reproduce the above copyright
+The above copyright notice and this permission notice shall be included in
-      notice, this list of conditions and the following disclaimer in the
+all copies or substantial portions of the Software.
-      documentation and/or other materials provided with the
-      distribution.
 
-    * Neither the name of the Hiroshima University nor the names of its
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-      contributors may be used to endorse or promote products derived
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-      from this software without specific prior written permission.
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
-
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+THE SOFTWARE.
-PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
-CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
-PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
-LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
-NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

README

@@ -1,37 +0,0 @@
-wrk - a HTTP benchmarking tool
-
-  wrk is a modern HTTP benchmarking tool capable of generating significant
-  load when run on a single multi-core CPU. It combines a multithreaded
-  design with scalable event notification systems such as epoll and kqueue.
-
-Basic Usage
-
-  wrk -t8 -c400 -d30 http://localhost:8080/index.html
-
-  This runs a benchmark for 30 seconds, using 8 threads, and keeping
-  400 HTTP connections open.
-
-  Output:
-
-  Running 30s test @ http://localhost:8080/index.html
-    8 threads and 400 connections
-    Thread Stats   Avg      Stdev     Max   +/- Stdev
-      Latency   439.75us  350.49us   7.60ms   92.88%
-      Req/Sec    61.13k     8.26k   72.00k    87.54%
-    10000088 requests in 19.87s, 3.42GB read
-  Requests/sec: 503396.23
-  Transfer/sec:    176.16MB
-
-Benchmarking Tips
-
-  The machine running wrk must have a sufficient number of ephemeral ports
-  available and closed sockets should be recycled quickly. To handle the
-  initial connection burst the server's listen(2) backlog should be greater
-  than the number of concurrent connections being tested.
-
-Acknowledgements
-
-  wrk contains code from a number of open source projects including the
-  'ae' event loop from redis, the nginx/joyent/node.js 'http-parser' and
-  the Tiny Mersenne Twister PRNG. Please consult the NOTICE file for
-  licensing details.

README.md

@@ -0,0 +1,85 @@
+# wrk - a HTTP benchmarking tool
+
+  wrk is a modern HTTP benchmarking tool capable of generating significant
+  load when run on a single multi-core CPU. It combines a multithreaded
+  design with scalable event notification systems such as epoll and kqueue.
+
+  An optional LuaJIT script can perform HTTP request generation, response
+  processing, and custom reporting. Details are available in SCRIPTING and
+  several examples are located in [scripts/](scripts/).
+
+## Basic Usage
+
+    wrk -t12 -c400 -d30s http://127.0.0.1:8080/index.html
+
+  This runs a benchmark for 30 seconds, using 12 threads, and keeping
+  400 HTTP connections open.
+
+  Output:
+
+    Running 30s test @ http://127.0.0.1:8080/index.html
+      12 threads and 400 connections
+      Thread Stats   Avg      Stdev     Max   +/- Stdev
+        Latency   635.91us    0.89ms  12.92ms   93.69%
+        Req/Sec    56.20k     8.07k   62.00k    86.54%
+      22464657 requests in 30.00s, 17.76GB read
+    Requests/sec: 748868.53
+    Transfer/sec:    606.33MB
+
+## Command Line Options
+
+    -c, --connections: total number of HTTP connections to keep open with
+                       each thread handling N = connections/threads
+
+    -d, --duration:    duration of the test, e.g. 2s, 2m, 2h
+
+    -t, --threads:     total number of threads to use
+
+    -s, --script:      LuaJIT script, see SCRIPTING
+
+    -H, --header:      HTTP header to add to request, e.g. "User-Agent: wrk"
+
+        --latency:     print detailed latency statistics
+
+        --timeout:     record a timeout if a response is not received within
+                       this amount of time.
+
+## Benchmarking Tips
+
+  The machine running wrk must have a sufficient number of ephemeral ports
+  available and closed sockets should be recycled quickly. To handle the
+  initial connection burst the server's listen(2) backlog should be greater
+  than the number of concurrent connections being tested.
+
+  A user script that only changes the HTTP method, path, adds headers or
+  a body, will have no performance impact. Per-request actions, particularly
+  building a new HTTP request, and use of response() will necessarily reduce
+  the amount of load that can be generated.
+
+## Acknowledgements
+
+  wrk contains code from a number of open source projects including the
+  'ae' event loop from redis, the nginx/joyent/node.js 'http-parser',
+  and Mike Pall's LuaJIT. Please consult the NOTICE file for licensing
+  details.
+
+## Cryptography Notice
+
+  This distribution includes cryptographic software. The country in
+  which you currently reside may have restrictions on the import,
+  possession, use, and/or re-export to another country, of encryption
+  software. BEFORE using any encryption software, please check your
+  country's laws, regulations and policies concerning the import,
+  possession, or use, and re-export of encryption software, to see if
+  this is permitted. See <http://www.wassenaar.org/> for more
+  information.
+
+  The U.S. Government Department of Commerce, Bureau of Industry and
+  Security (BIS), has classified this software as Export Commodity
+  Control Number (ECCN) 5D002.C.1, which includes information security
+  software using or performing cryptographic functions with symmetric
+  algorithms. The form and manner of this distribution makes it
+  eligible for export under the License Exception ENC Technology
+  Software Unrestricted (TSU) exception (see the BIS Export
+  Administration Regulations, Section 740.13) for both object code and
+  source code.

SCRIPTING

@@ -0,0 +1,117 @@
+Overview
+
+  wrk supports executing a LuaJIT script during three distinct phases: setup,
+  running, and done. Each wrk thread has an independent scripting environment
+  and the setup & done phases execute in a separate environment which does
+  not participate in the running phase.
+
+  The public Lua API consists of a global table and a number of global
+  functions:
+
+  wrk = {
+    scheme  = "http",
+    host    = "localhost",
+    port    = nil,
+    method  = "GET",
+    path    = "/",
+    headers = {},
+    body    = nil,
+    thread  = <userdata>,
+  }
+
+  function wrk.format(method, path, headers, body)
+
+    wrk.format returns a HTTP request string containing the passed parameters
+    merged with values from the wrk table.
+
+  function wrk.lookup(host, service)
+
+    wrk.lookup returns a table containing all known addresses for the host
+    and service pair. This corresponds to the POSIX getaddrinfo() function.
+
+  function wrk.connect(addr)
+
+    wrk.connect returns true if the address can be connected to, otherwise
+    it returns false. The address must be one returned from wrk.lookup().
+
+  The following globals are optional, and if defined must be functions:
+
+    global setup    -- called during thread setup
+    global init     -- called when the thread is starting
+    global delay    -- called to get the request delay
+    global request  -- called to generate the HTTP request
+    global response -- called with HTTP response data
+    global done     -- called with results of run
+
+Setup
+
+  function setup(thread)
+
+  The setup phase begins after the target IP address has been resolved and all
+  threads have been initialized but not yet started.
+
+  setup() is called once for each thread and receives a userdata object
+  representing the thread.
+
+    thread.addr             - get or set the thread's server address
+    thread:get(name)        - get the value of a global in the thread's env
+    thread:set(name, value) - set the value of a global in the thread's env
+    thread:stop()           - stop the thread
+
+  Only boolean, nil, number, and string values or tables of the same may be
+  transfered via get()/set() and thread:stop() can only be called while the
+  thread is running.
+
+Running
+
+  function init(args)
+  function delay()
+  function request()
+  function response(status, headers, body)
+
+  The running phase begins with a single call to init(), followed by
+  a call to request() and response() for each request cycle.
+
+  The init() function receives any extra command line arguments for the
+  script which must be separated from wrk arguments with "--".
+
+  delay() returns the number of milliseconds to delay sending the next
+  request.
+
+  request() returns a string containing the HTTP request. Building a new
+  request each time is expensive, when testing a high performance server
+  one solution is to pre-generate all requests in init() and do a quick
+  lookup in request().
+
+  response() is called with the HTTP response status, headers, and body.
+  Parsing the headers and body is expensive, so if the response global is
+  nil after the call to init() wrk will ignore the headers and body.
+
+Done
+
+  function done(summary, latency, requests)
+
+  The done() function receives a table containing result data, and two
+  statistics objects representing the per-request latency and per-thread
+  request rate. Duration and latency are microsecond values and rate is
+  measured in requests per second.
+
+  latency.min              -- minimum value seen
+  latency.max              -- maximum value seen
+  latency.mean             -- average value seen
+  latency.stdev            -- standard deviation
+  latency:percentile(99.0) -- 99th percentile value
+  latency(i)               -- raw value and count
+
+  summary = {
+    duration = N,  -- run duration in microseconds
+    requests = N,  -- total completed requests
+    bytes    = N,  -- total bytes received
+    errors   = {
+      connect = N, -- total socket connection errors
+      read    = N, -- total socket read errors
+      write   = N, -- total socket write errors
+      status  = N, -- total HTTP status codes > 399
+      timeout = N  -- total request timeouts
+    }
+  }

scripts/addr.lua

@@ -0,0 +1,22 @@
+-- example script that demonstrates use of setup() to pass
+-- a random server address to each thread
+
+local addrs = nil
+
+function setup(thread)
+   if not addrs then
+      addrs = wrk.lookup(wrk.host, wrk.port or "http")
+      for i = #addrs, 1, -1 do
+         if not wrk.connect(addrs[i]) then
+            table.remove(addrs, i)
+         end
+      end
+   end
+
+   thread.addr = addrs[math.random(#addrs)]
+end
+
+function init(args)
+   local msg = "thread addr: %s"
+   print(msg:format(wrk.thread.addr))
+end

scripts/auth.lua

@@ -0,0 +1,18 @@
+-- example script that demonstrates response handling and
+-- retrieving an authentication token to set on all future
+-- requests
+
+token = nil
+path  = "/authenticate"
+
+request = function()
+   return wrk.format("GET", path)
+end
+
+response = function(status, headers, body)
+   if not token and status == 200 then
+      token = headers["X-Token"]
+      path  = "/resource"
+      wrk.headers["X-Token"] = token
+   end
+end

scripts/counter.lua

@@ -0,0 +1,14 @@
+-- example dynamic request script which demonstrates changing
+-- the request path and a header for each request
+-------------------------------------------------------------
+-- NOTE: each wrk thread has an independent Lua scripting
+-- context and thus there will be one counter per thread
+
+counter = 0
+
+request = function()
+   path = "/" .. counter
+   wrk.headers["X-Counter"] = counter
+   counter = counter + 1
+   return wrk.format(nil, path)
+end

scripts/delay.lua

@@ -0,0 +1,6 @@
+-- example script that demonstrates adding a random
+-- 10-50ms delay before each request
+
+function delay()
+   return math.random(10, 50)
+end

scripts/pipeline.lua

@@ -0,0 +1,14 @@
+-- example script demonstrating HTTP pipelining
+
+init = function(args)
+   local r = {}
+   r[1] = wrk.format(nil, "/?foo")
+   r[2] = wrk.format(nil, "/?bar")
+   r[3] = wrk.format(nil, "/?baz")
+
+   req = table.concat(r)
+end
+
+request = function()
+   return req
+end

scripts/post.lua

@@ -0,0 +1,6 @@
+-- example HTTP POST script which demonstrates setting the
+-- HTTP method, body, and adding a header
+
+wrk.method = "POST"
+wrk.body   = "foo=bar&baz=quux"
+wrk.headers["Content-Type"] = "application/x-www-form-urlencoded"

scripts/report.lua

@@ -0,0 +1,10 @@
+-- example reporting script which demonstrates a custom
+-- done() function that prints latency percentiles as CSV
+
+done = function(summary, latency, requests)
+   io.write("------------------------------\n")
+   for _, p in pairs({ 50, 90, 99, 99.999 }) do
+      n = latency:percentile(p)
+      io.write(string.format("%g%%,%d\n", p, n))
+   end
+end

scripts/setup.lua

@@ -0,0 +1,38 @@
+-- example script that demonstrates use of setup() to pass
+-- data to and from the threads
+
+local counter = 1
+local threads = {}
+
+function setup(thread)
+   thread:set("id", counter)
+   table.insert(threads, thread)
+   counter = counter + 1
+end
+
+function init(args)
+   requests  = 0
+   responses = 0
+
+   local msg = "thread %d created"
+   print(msg:format(id))
+end
+
+function request()
+   requests = requests + 1
+   return wrk.request()
+end
+
+function response(status, headers, body)
+   responses = responses + 1
+end
+
+function done(summary, latency, requests)
+   for index, thread in ipairs(threads) do
+      local id        = thread:get("id")
+      local requests  = thread:get("requests")
+      local responses = thread:get("responses")
+      local msg = "thread %d made %d requests and got %d responses"
+      print(msg:format(id, requests, responses))
+   end
+end

scripts/stop.lua

@@ -0,0 +1,10 @@
+-- example script that demonstrates use of thread:stop()
+
+local counter = 1
+
+function response()
+   if counter == 100 then
+      wrk.thread:stop()
+   end
+   counter = counter + 1
+end

src/ae.c

@@ -91,6 +91,36 @@ err:
     return NULL;
 }
 
+/* Return the current set size. */
+int aeGetSetSize(aeEventLoop *eventLoop) {
+    return eventLoop->setsize;
+}
+
+/* Resize the maximum set size of the event loop.
+ * If the requested set size is smaller than the current set size, but
+ * there is already a file descriptor in use that is >= the requested
+ * set size minus one, AE_ERR is returned and the operation is not
+ * performed at all.
+ *
+ * Otherwise AE_OK is returned and the operation is successful. */
+int aeResizeSetSize(aeEventLoop *eventLoop, int setsize) {
+    int i;
+
+    if (setsize == eventLoop->setsize) return AE_OK;
+    if (eventLoop->maxfd >= setsize) return AE_ERR;
+    if (aeApiResize(eventLoop,setsize) == -1) return AE_ERR;
+
+    eventLoop->events = zrealloc(eventLoop->events,sizeof(aeFileEvent)*setsize);
+    eventLoop->fired = zrealloc(eventLoop->fired,sizeof(aeFiredEvent)*setsize);
+    eventLoop->setsize = setsize;
+
+    /* Make sure that if we created new slots, they are initialized with
+     * an AE_NONE mask. */
+    for (i = eventLoop->maxfd+1; i < setsize; i++)
+        eventLoop->events[i].mask = AE_NONE;
+    return AE_OK;
+}
+
 void aeDeleteEventLoop(aeEventLoop *eventLoop) {
     aeApiFree(eventLoop);
     zfree(eventLoop->events);
@@ -126,8 +156,9 @@ void aeDeleteFileEvent(aeEventLoop *eventLoop, int fd, int mask)
 {
     if (fd >= eventLoop->setsize) return;
     aeFileEvent *fe = &eventLoop->events[fd];
-
     if (fe->mask == AE_NONE) return;
+
+    aeApiDelEvent(eventLoop, fd, mask);
     fe->mask = fe->mask & (~mask);
     if (fd == eventLoop->maxfd && fe->mask == AE_NONE) {
         /* Update the max fd */
@@ -137,7 +168,6 @@ void aeDeleteFileEvent(aeEventLoop *eventLoop, int fd, int mask)
             if (eventLoop->events[j].mask != AE_NONE) break;
         eventLoop->maxfd = j;
     }
-    aeApiDelEvent(eventLoop, fd, mask);
 }
 
 int aeGetFileEvents(aeEventLoop *eventLoop, int fd) {
@@ -191,21 +221,12 @@ long long aeCreateTimeEvent(aeEventLoop *eventLoop, long long milliseconds,
 
 int aeDeleteTimeEvent(aeEventLoop *eventLoop, long long id)
 {
-    aeTimeEvent *te, *prev = NULL;
+    aeTimeEvent *te = eventLoop->timeEventHead;
-
-    te = eventLoop->timeEventHead;
     while(te) {
         if (te->id == id) {
-            if (prev == NULL)
+            te->id = AE_DELETED_EVENT_ID;
-                eventLoop->timeEventHead = te->next;
-            else
-                prev->next = te->next;
-            if (te->finalizerProc)
-                te->finalizerProc(eventLoop, te->clientData);
-            zfree(te);
             return AE_OK;
         }
-        prev = te;
         te = te->next;
     }
     return AE_ERR; /* NO event with the specified ID found */
@@ -240,7 +261,7 @@ static aeTimeEvent *aeSearchNearestTimer(aeEventLoop *eventLoop)
 /* Process time events */
 static int processTimeEvents(aeEventLoop *eventLoop) {
     int processed = 0;
-    aeTimeEvent *te;
+    aeTimeEvent *te, *prev;
     long long maxId;
     time_t now = time(NULL);
 
@@ -261,12 +282,32 @@ static int processTimeEvents(aeEventLoop *eventLoop) {
     }
     eventLoop->lastTime = now;
 
+    prev = NULL;
     te = eventLoop->timeEventHead;
     maxId = eventLoop->timeEventNextId-1;
     while(te) {
         long now_sec, now_ms;
         long long id;
 
+        /* Remove events scheduled for deletion. */
+        if (te->id == AE_DELETED_EVENT_ID) {
+            aeTimeEvent *next = te->next;
+            if (prev == NULL)
+                eventLoop->timeEventHead = te->next;
+            else
+                prev->next = te->next;
+            if (te->finalizerProc)
+                te->finalizerProc(eventLoop, te->clientData);
+            zfree(te);
+            te = next;
+            continue;
+        }
+
+        /* Make sure we don't process time events created by time events in
+         * this iteration. Note that this check is currently useless: we always
+         * add new timers on the head, however if we change the implementation
+         * detail, this check may be useful again: we keep it here for future
+         * defense. */
         if (te->id > maxId) {
             te = te->next;
             continue;
@@ -280,28 +321,14 @@ static int processTimeEvents(aeEventLoop *eventLoop) {
             id = te->id;
             retval = te->timeProc(eventLoop, id, te->clientData);
             processed++;
-            /* After an event is processed our time event list may
-             * no longer be the same, so we restart from head.
-             * Still we make sure to don't process events registered
-             * by event handlers itself in order to don't loop forever.
-             * To do so we saved the max ID we want to handle.
-             *
-             * FUTURE OPTIMIZATIONS:
-             * Note that this is NOT great algorithmically. Redis uses
-             * a single time event so it's not a problem but the right
-             * way to do this is to add the new elements on head, and
-             * to flag deleted elements in a special way for later
-             * deletion (putting references to the nodes to delete into
-             * another linked list). */
             if (retval != AE_NOMORE) {
                 aeAddMillisecondsToNow(retval,&te->when_sec,&te->when_ms);
             } else {
-                aeDeleteTimeEvent(eventLoop, id);
+                te->id = AE_DELETED_EVENT_ID;
             }
-            te = eventLoop->timeEventHead;
-        } else {
-            te = te->next;
         }
+        prev = te;
+        te = te->next;
     }
     return processed;
 }
@@ -309,7 +336,7 @@ static int processTimeEvents(aeEventLoop *eventLoop) {
 /* Process every pending time event, then every pending file event
  * (that may be registered by time event callbacks just processed).
  * Without special flags the function sleeps until some file event
- * fires, or when the next time event occurrs (if any).
+ * fires, or when the next time event occurs (if any).
  *
  * If flags is 0, the function does nothing and returns.
  * if flags has AE_ALL_EVENTS set, all the kind of events are processed.
@@ -341,22 +368,25 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags)
         if (shortest) {
             long now_sec, now_ms;
 
-            /* Calculate the time missing for the nearest
-             * timer to fire. */
             aeGetTime(&now_sec, &now_ms);
             tvp = &tv;
-            tvp->tv_sec = shortest->when_sec - now_sec;
+
-            if (shortest->when_ms < now_ms) {
+            /* How many milliseconds we need to wait for the next
-                tvp->tv_usec = ((shortest->when_ms+1000) - now_ms)*1000;
+             * time event to fire? */
-                tvp->tv_sec --;
+            long long ms =
+                (shortest->when_sec - now_sec)*1000 +
+                shortest->when_ms - now_ms;
+
+            if (ms > 0) {
+                tvp->tv_sec = ms/1000;
+                tvp->tv_usec = (ms % 1000)*1000;
             } else {
-                tvp->tv_usec = (shortest->when_ms - now_ms)*1000;
+                tvp->tv_sec = 0;
+                tvp->tv_usec = 0;
             }
-            if (tvp->tv_sec < 0) tvp->tv_sec = 0;
-            if (tvp->tv_usec < 0) tvp->tv_usec = 0;
         } else {
             /* If we have to check for events but need to return
-             * ASAP because of AE_DONT_WAIT we need to se the timeout
+             * ASAP because of AE_DONT_WAIT we need to set the timeout
              * to zero */
             if (flags & AE_DONT_WAIT) {
                 tv.tv_sec = tv.tv_usec = 0;
@@ -395,7 +425,7 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags)
     return processed; /* return the number of processed file/time events */
 }
 
-/* Wait for millseconds until the given file descriptor becomes
+/* Wait for milliseconds until the given file descriptor becomes
  * writable/readable/exception */
 int aeWait(int fd, int mask, long long milliseconds) {
     struct pollfd pfd;

src/ae.h

@@ -33,6 +33,8 @@
 #ifndef __AE_H__
 #define __AE_H__
 
+#include <time.h>
+
 #define AE_OK 0
 #define AE_ERR -1
 
@@ -46,6 +48,7 @@
 #define AE_DONT_WAIT 4
 
 #define AE_NOMORE -1
+#define AE_DELETED_EVENT_ID -1
 
 /* Macros */
 #define AE_NOTUSED(V) ((void) V)
@@ -114,5 +117,7 @@ int aeWait(int fd, int mask, long long milliseconds);
 void aeMain(aeEventLoop *eventLoop);
 char *aeGetApiName(void);
 void aeSetBeforeSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *beforesleep);
+int aeGetSetSize(aeEventLoop *eventLoop);
+int aeResizeSetSize(aeEventLoop *eventLoop, int setsize);
 
 #endif

src/ae_epoll.c

@@ -45,7 +45,7 @@ static int aeApiCreate(aeEventLoop *eventLoop) {
         zfree(state);
         return -1;
     }
-    state->epfd = epoll_create(1024); /* 1024 is just an hint for the kernel */
+    state->epfd = epoll_create(1024); /* 1024 is just a hint for the kernel */
     if (state->epfd == -1) {
         zfree(state->events);
         zfree(state);
@@ -55,6 +55,13 @@ static int aeApiCreate(aeEventLoop *eventLoop) {
     return 0;
 }
 
+static int aeApiResize(aeEventLoop *eventLoop, int setsize) {
+    aeApiState *state = eventLoop->apidata;
+
+    state->events = zrealloc(state->events, sizeof(struct epoll_event)*setsize);
+    return 0;
+}
+
 static void aeApiFree(aeEventLoop *eventLoop) {
     aeApiState *state = eventLoop->apidata;
 
@@ -65,7 +72,7 @@ static void aeApiFree(aeEventLoop *eventLoop) {
 
 static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) {
     aeApiState *state = eventLoop->apidata;
-    struct epoll_event ee;
+    struct epoll_event ee = {0}; /* avoid valgrind warning */
     /* If the fd was already monitored for some event, we need a MOD
      * operation. Otherwise we need an ADD operation. */
     int op = eventLoop->events[fd].mask == AE_NONE ?
@@ -75,7 +82,6 @@ static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) {
     mask |= eventLoop->events[fd].mask; /* Merge old events */
     if (mask & AE_READABLE) ee.events |= EPOLLIN;
     if (mask & AE_WRITABLE) ee.events |= EPOLLOUT;
-    ee.data.u64 = 0; /* avoid valgrind warning */
     ee.data.fd = fd;
     if (epoll_ctl(state->epfd,op,fd,&ee) == -1) return -1;
     return 0;
@@ -83,13 +89,12 @@ static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) {
 
 static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int delmask) {
     aeApiState *state = eventLoop->apidata;
-    struct epoll_event ee;
+    struct epoll_event ee = {0}; /* avoid valgrind warning */
     int mask = eventLoop->events[fd].mask & (~delmask);
 
     ee.events = 0;
     if (mask & AE_READABLE) ee.events |= EPOLLIN;
     if (mask & AE_WRITABLE) ee.events |= EPOLLOUT;
-    ee.data.u64 = 0; /* avoid valgrind warning */
     ee.data.fd = fd;
     if (mask != AE_NONE) {
         epoll_ctl(state->epfd,EPOLL_CTL_MOD,fd,&ee);

src/ae_evport.c

@@ -50,15 +50,15 @@ static int evport_debug = 0;
  * aeApiPoll, the corresponding file descriptors become dissociated from the
  * port.  This is necessary because poll events are level-triggered, so if the
  * fd didn't become dissociated, it would immediately fire another event since
- * the underlying state hasn't changed yet.  We must reassociate the file
+ * the underlying state hasn't changed yet.  We must re-associate the file
  * descriptor, but only after we know that our caller has actually read from it.
  * The ae API does not tell us exactly when that happens, but we do know that
  * it must happen by the time aeApiPoll is called again.  Our solution is to
- * keep track of the last fds returned by aeApiPoll and reassociate them next
+ * keep track of the last fds returned by aeApiPoll and re-associate them next
  * time aeApiPoll is invoked.
  *
  * To summarize, in this module, each fd association is EITHER (a) represented
- * only via the in-kernel assocation OR (b) represented by pending_fds and
+ * only via the in-kernel association OR (b) represented by pending_fds and
  * pending_masks.  (b) is only true for the last fds we returned from aeApiPoll,
  * and only until we enter aeApiPoll again (at which point we restore the
  * in-kernel association).
@@ -94,6 +94,11 @@ static int aeApiCreate(aeEventLoop *eventLoop) {
     return 0;
 }
 
+static int aeApiResize(aeEventLoop *eventLoop, int setsize) {
+    /* Nothing to resize here. */
+    return 0;
+}
+
 static void aeApiFree(aeEventLoop *eventLoop) {
     aeApiState *state = eventLoop->apidata;
 
@@ -164,7 +169,7 @@ static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) {
          * This fd was recently returned from aeApiPoll.  It should be safe to
          * assume that the consumer has processed that poll event, but we play
          * it safer by simply updating pending_mask.  The fd will be
-         * reassociated as usual when aeApiPoll is called again.
+         * re-associated as usual when aeApiPoll is called again.
          */
         if (evport_debug)
             fprintf(stderr, "aeApiAddEvent: adding to pending fd %d\n", fd);
@@ -228,7 +233,7 @@ static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) {
          * ENOMEM is a potentially transient condition, but the kernel won't
          * generally return it unless things are really bad.  EAGAIN indicates
          * we've reached an resource limit, for which it doesn't make sense to
-         * retry (counterintuitively).  All other errors indicate a bug.  In any
+         * retry (counter-intuitively).  All other errors indicate a bug.  In any
          * of these cases, the best we can do is to abort.
          */
         abort(); /* will not return */
@@ -243,7 +248,7 @@ static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) {
     port_event_t event[MAX_EVENT_BATCHSZ];
 
     /*
-     * If we've returned fd events before, we must reassociate them with the
+     * If we've returned fd events before, we must re-associate them with the
      * port now, before calling port_get().  See the block comment at the top of
      * this file for an explanation of why.
      */

src/ae_kqueue.c

@@ -54,8 +54,14 @@ static int aeApiCreate(aeEventLoop *eventLoop) {
         return -1;
     }
     eventLoop->apidata = state;
-    
+    return 0;
-    return 0;    
+}
+
+static int aeApiResize(aeEventLoop *eventLoop, int setsize) {
+    aeApiState *state = eventLoop->apidata;
+
+    state->events = zrealloc(state->events, sizeof(struct kevent)*setsize);
+    return 0;
 }
 
 static void aeApiFree(aeEventLoop *eventLoop) {
@@ -69,7 +75,7 @@ static void aeApiFree(aeEventLoop *eventLoop) {
 static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) {
     aeApiState *state = eventLoop->apidata;
     struct kevent ke;
-    
+
     if (mask & AE_READABLE) {
         EV_SET(&ke, fd, EVFILT_READ, EV_ADD, 0, 0, NULL);
         if (kevent(state->kqfd, &ke, 1, NULL, 0, NULL) == -1) return -1;
@@ -112,16 +118,16 @@ static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) {
 
     if (retval > 0) {
         int j;
-        
+
         numevents = retval;
         for(j = 0; j < numevents; j++) {
             int mask = 0;
             struct kevent *e = state->events+j;
-            
+
             if (e->filter == EVFILT_READ) mask |= AE_READABLE;
             if (e->filter == EVFILT_WRITE) mask |= AE_WRITABLE;
-            eventLoop->fired[j].fd = e->ident; 
+            eventLoop->fired[j].fd = e->ident;
-            eventLoop->fired[j].mask = mask;           
+            eventLoop->fired[j].mask = mask;
         }
     }
     return numevents;

src/ae_select.c

@@ -29,6 +29,7 @@
  */
 
 
+#include <sys/select.h>
 #include <string.h>
 
 typedef struct aeApiState {
@@ -48,6 +49,12 @@ static int aeApiCreate(aeEventLoop *eventLoop) {
     return 0;
 }
 
+static int aeApiResize(aeEventLoop *eventLoop, int setsize) {
+    /* Just ensure we have enough room in the fd_set type. */
+    if (setsize >= FD_SETSIZE) return -1;
+    return 0;
+}
+
 static void aeApiFree(aeEventLoop *eventLoop) {
     zfree(eventLoop->apidata);
 }

src/atomicvar.h

@@ -0,0 +1,133 @@
+/* This file implements atomic counters using __atomic or __sync macros if
+ * available, otherwise synchronizing different threads using a mutex.
+ *
+ * The exported interaface is composed of three macros:
+ *
+ * atomicIncr(var,count) -- Increment the atomic counter
+ * atomicGetIncr(var,oldvalue_var,count) -- Get and increment the atomic counter
+ * atomicDecr(var,count) -- Decrement the atomic counter
+ * atomicGet(var,dstvar) -- Fetch the atomic counter value
+ * atomicSet(var,value)  -- Set the atomic counter value
+ *
+ * The variable 'var' should also have a declared mutex with the same
+ * name and the "_mutex" postfix, for instance:
+ *
+ *  long myvar;
+ *  pthread_mutex_t myvar_mutex;
+ *  atomicSet(myvar,12345);
+ *
+ * If atomic primitives are availble (tested in config.h) the mutex
+ * is not used.
+ *
+ * Never use return value from the macros, instead use the AtomicGetIncr()
+ * if you need to get the current value and increment it atomically, like
+ * in the followign example:
+ *
+ *  long oldvalue;
+ *  atomicGetIncr(myvar,oldvalue,1);
+ *  doSomethingWith(oldvalue);
+ *
+ * ----------------------------------------------------------------------------
+ *
+ * Copyright (c) 2015, Salvatore Sanfilippo <antirez at gmail dot com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <pthread.h>
+
+#ifndef __ATOMIC_VAR_H
+#define __ATOMIC_VAR_H
+
+/* To test Redis with Helgrind (a Valgrind tool) it is useful to define
+ * the following macro, so that __sync macros are used: those can be detected
+ * by Helgrind (even if they are less efficient) so that no false positive
+ * is reported. */
+// #define __ATOMIC_VAR_FORCE_SYNC_MACROS
+
+#if !defined(__ATOMIC_VAR_FORCE_SYNC_MACROS) && defined(__ATOMIC_RELAXED) && !defined(__sun) && (!defined(__clang__) || !defined(__APPLE__) || __apple_build_version__ > 4210057)
+/* Implementation using __atomic macros. */
+
+#define atomicIncr(var,count) __atomic_add_fetch(&var,(count),__ATOMIC_RELAXED)
+#define atomicGetIncr(var,oldvalue_var,count) do { \
+    oldvalue_var = __atomic_fetch_add(&var,(count),__ATOMIC_RELAXED); \
+} while(0)
+#define atomicDecr(var,count) __atomic_sub_fetch(&var,(count),__ATOMIC_RELAXED)
+#define atomicGet(var,dstvar) do { \
+    dstvar = __atomic_load_n(&var,__ATOMIC_RELAXED); \
+} while(0)
+#define atomicSet(var,value) __atomic_store_n(&var,value,__ATOMIC_RELAXED)
+#define REDIS_ATOMIC_API "atomic-builtin"
+
+#elif defined(HAVE_ATOMIC)
+/* Implementation using __sync macros. */
+
+#define atomicIncr(var,count) __sync_add_and_fetch(&var,(count))
+#define atomicGetIncr(var,oldvalue_var,count) do { \
+    oldvalue_var = __sync_fetch_and_add(&var,(count)); \
+} while(0)
+#define atomicDecr(var,count) __sync_sub_and_fetch(&var,(count))
+#define atomicGet(var,dstvar) do { \
+    dstvar = __sync_sub_and_fetch(&var,0); \
+} while(0)
+#define atomicSet(var,value) do { \
+    while(!__sync_bool_compare_and_swap(&var,var,value)); \
+} while(0)
+#define REDIS_ATOMIC_API "sync-builtin"
+
+#else
+/* Implementation using pthread mutex. */
+
+#define atomicIncr(var,count) do { \
+    pthread_mutex_lock(&var ## _mutex); \
+    var += (count); \
+    pthread_mutex_unlock(&var ## _mutex); \
+} while(0)
+#define atomicGetIncr(var,oldvalue_var,count) do { \
+    pthread_mutex_lock(&var ## _mutex); \
+    oldvalue_var = var; \
+    var += (count); \
+    pthread_mutex_unlock(&var ## _mutex); \
+} while(0)
+#define atomicDecr(var,count) do { \
+    pthread_mutex_lock(&var ## _mutex); \
+    var -= (count); \
+    pthread_mutex_unlock(&var ## _mutex); \
+} while(0)
+#define atomicGet(var,dstvar) do { \
+    pthread_mutex_lock(&var ## _mutex); \
+    dstvar = var; \
+    pthread_mutex_unlock(&var ## _mutex); \
+} while(0)
+#define atomicSet(var,value) do { \
+    pthread_mutex_lock(&var ## _mutex); \
+    var = value; \
+    pthread_mutex_unlock(&var ## _mutex); \
+} while(0)
+#define REDIS_ATOMIC_API "pthread-mutex"
+
+#endif
+#endif /* __ATOMIC_VAR_H */

src/config.h

@@ -5,9 +5,13 @@
 #define HAVE_KQUEUE
 #elif defined(__linux__)
 #define HAVE_EPOLL
-#define _POSIX_C_SOURCE 200809L
 #elif defined (__sun)
 #define HAVE_EVPORT
+#define _XPG6
+#define __EXTENSIONS__
+#include <stropts.h>
+#include <sys/filio.h>
+#include <sys/time.h>
 #endif
 
 #endif /* CONFIG_H */

src/http_parser.h

@@ -24,13 +24,15 @@
 extern "C" {
 #endif
 
+/* Also update SONAME in the Makefile whenever you change these. */
 #define HTTP_PARSER_VERSION_MAJOR 2
-#define HTTP_PARSER_VERSION_MINOR 1
+#define HTTP_PARSER_VERSION_MINOR 7
+#define HTTP_PARSER_VERSION_PATCH 1
 
-#include <sys/types.h>
-#if defined(_WIN32) && !defined(__MINGW32__) && (!defined(_MSC_VER) || _MSC_VER<1600)
-#include <BaseTsd.h>
 #include <stddef.h>
+#if defined(_WIN32) && !defined(__MINGW32__) && \
+  (!defined(_MSC_VER) || _MSC_VER<1600) && !defined(__WINE__)
+#include <BaseTsd.h>
 typedef __int8 int8_t;
 typedef unsigned __int8 uint8_t;
 typedef __int16 int16_t;
@@ -50,9 +52,16 @@ typedef unsigned __int64 uint64_t;
 # define HTTP_PARSER_STRICT 1
 #endif
 
-/* Maximium header size allowed */
+/* Maximium header size allowed. If the macro is not defined
-#define HTTP_MAX_HEADER_SIZE (80*1024)
+ * before including this header then the default is used. To
-
+ * change the maximum header size, define the macro in the build
+ * environment (e.g. -DHTTP_MAX_HEADER_SIZE=<value>). To remove
+ * the effective limit on the size of the header, define the macro
+ * to a very large number (e.g. -DHTTP_MAX_HEADER_SIZE=0x7fffffff)
+ */
+#ifndef HTTP_MAX_HEADER_SIZE
+# define HTTP_MAX_HEADER_SIZE (80*1024)
+#endif
 
 typedef struct http_parser http_parser;
 typedef struct http_parser_settings http_parser_settings;
@@ -67,7 +76,12 @@ typedef struct http_parser_settings http_parser_settings;
  * HEAD request which may contain 'Content-Length' or 'Transfer-Encoding:
  * chunked' headers that indicate the presence of a body.
  *
- * http_data_cb does not return data chunks. It will be call arbitrarally
+ * Returning `2` from on_headers_complete will tell parser that it should not
+ * expect neither a body nor any futher responses on this connection. This is
+ * useful for handling responses to a CONNECT request which may not contain
+ * `Upgrade` or `Connection: upgrade` headers.
+ *
+ * http_data_cb does not return data chunks. It will be called arbitrarily
  * many times for each string. E.G. you might get 10 callbacks for "on_url"
  * each providing just a few characters more data.
  */
@@ -75,6 +89,76 @@ typedef int (*http_data_cb) (http_parser*, const char *at, size_t length);
 typedef int (*http_cb) (http_parser*);
 
 
+/* Status Codes */
+#define HTTP_STATUS_MAP(XX)                                                 \
+  XX(100, CONTINUE,                        Continue)                        \
+  XX(101, SWITCHING_PROTOCOLS,             Switching Protocols)             \
+  XX(102, PROCESSING,                      Processing)                      \
+  XX(200, OK,                              OK)                              \
+  XX(201, CREATED,                         Created)                         \
+  XX(202, ACCEPTED,                        Accepted)                        \
+  XX(203, NON_AUTHORITATIVE_INFORMATION,   Non-Authoritative Information)   \
+  XX(204, NO_CONTENT,                      No Content)                      \
+  XX(205, RESET_CONTENT,                   Reset Content)                   \
+  XX(206, PARTIAL_CONTENT,                 Partial Content)                 \
+  XX(207, MULTI_STATUS,                    Multi-Status)                    \
+  XX(208, ALREADY_REPORTED,                Already Reported)                \
+  XX(226, IM_USED,                         IM Used)                         \
+  XX(300, MULTIPLE_CHOICES,                Multiple Choices)                \
+  XX(301, MOVED_PERMANENTLY,               Moved Permanently)               \
+  XX(302, FOUND,                           Found)                           \
+  XX(303, SEE_OTHER,                       See Other)                       \
+  XX(304, NOT_MODIFIED,                    Not Modified)                    \
+  XX(305, USE_PROXY,                       Use Proxy)                       \
+  XX(307, TEMPORARY_REDIRECT,              Temporary Redirect)              \
+  XX(308, PERMANENT_REDIRECT,              Permanent Redirect)              \
+  XX(400, BAD_REQUEST,                     Bad Request)                     \
+  XX(401, UNAUTHORIZED,                    Unauthorized)                    \
+  XX(402, PAYMENT_REQUIRED,                Payment Required)                \
+  XX(403, FORBIDDEN,                       Forbidden)                       \
+  XX(404, NOT_FOUND,                       Not Found)                       \
+  XX(405, METHOD_NOT_ALLOWED,              Method Not Allowed)              \
+  XX(406, NOT_ACCEPTABLE,                  Not Acceptable)                  \
+  XX(407, PROXY_AUTHENTICATION_REQUIRED,   Proxy Authentication Required)   \
+  XX(408, REQUEST_TIMEOUT,                 Request Timeout)                 \
+  XX(409, CONFLICT,                        Conflict)                        \
+  XX(410, GONE,                            Gone)                            \
+  XX(411, LENGTH_REQUIRED,                 Length Required)                 \
+  XX(412, PRECONDITION_FAILED,             Precondition Failed)             \
+  XX(413, PAYLOAD_TOO_LARGE,               Payload Too Large)               \
+  XX(414, URI_TOO_LONG,                    URI Too Long)                    \
+  XX(415, UNSUPPORTED_MEDIA_TYPE,          Unsupported Media Type)          \
+  XX(416, RANGE_NOT_SATISFIABLE,           Range Not Satisfiable)           \
+  XX(417, EXPECTATION_FAILED,              Expectation Failed)              \
+  XX(421, MISDIRECTED_REQUEST,             Misdirected Request)             \
+  XX(422, UNPROCESSABLE_ENTITY,            Unprocessable Entity)            \
+  XX(423, LOCKED,                          Locked)                          \
+  XX(424, FAILED_DEPENDENCY,               Failed Dependency)               \
+  XX(426, UPGRADE_REQUIRED,                Upgrade Required)                \
+  XX(428, PRECONDITION_REQUIRED,           Precondition Required)           \
+  XX(429, TOO_MANY_REQUESTS,               Too Many Requests)               \
+  XX(431, REQUEST_HEADER_FIELDS_TOO_LARGE, Request Header Fields Too Large) \
+  XX(451, UNAVAILABLE_FOR_LEGAL_REASONS,   Unavailable For Legal Reasons)   \
+  XX(500, INTERNAL_SERVER_ERROR,           Internal Server Error)           \
+  XX(501, NOT_IMPLEMENTED,                 Not Implemented)                 \
+  XX(502, BAD_GATEWAY,                     Bad Gateway)                     \
+  XX(503, SERVICE_UNAVAILABLE,             Service Unavailable)             \
+  XX(504, GATEWAY_TIMEOUT,                 Gateway Timeout)                 \
+  XX(505, HTTP_VERSION_NOT_SUPPORTED,      HTTP Version Not Supported)      \
+  XX(506, VARIANT_ALSO_NEGOTIATES,         Variant Also Negotiates)         \
+  XX(507, INSUFFICIENT_STORAGE,            Insufficient Storage)            \
+  XX(508, LOOP_DETECTED,                   Loop Detected)                   \
+  XX(510, NOT_EXTENDED,                    Not Extended)                    \
+  XX(511, NETWORK_AUTHENTICATION_REQUIRED, Network Authentication Required) \
+
+enum http_status
+  {
+#define XX(num, name, string) HTTP_STATUS_##name = num,
+  HTTP_STATUS_MAP(XX)
+#undef XX
+  };
+
+
 /* Request Methods */
 #define HTTP_METHOD_MAP(XX)         \
   XX(0,  DELETE,      DELETE)       \
@@ -86,7 +170,7 @@ typedef int (*http_cb) (http_parser*);
   XX(5,  CONNECT,     CONNECT)      \
   XX(6,  OPTIONS,     OPTIONS)      \
   XX(7,  TRACE,       TRACE)        \
-  /* webdav */                      \
+  /* WebDAV */                      \
   XX(8,  COPY,        COPY)         \
   XX(9,  LOCK,        LOCK)         \
   XX(10, MKCOL,       MKCOL)        \
@@ -95,19 +179,28 @@ typedef int (*http_cb) (http_parser*);
   XX(13, PROPPATCH,   PROPPATCH)    \
   XX(14, SEARCH,      SEARCH)       \
   XX(15, UNLOCK,      UNLOCK)       \
+  XX(16, BIND,        BIND)         \
+  XX(17, REBIND,      REBIND)       \
+  XX(18, UNBIND,      UNBIND)       \
+  XX(19, ACL,         ACL)          \
   /* subversion */                  \
-  XX(16, REPORT,      REPORT)       \
+  XX(20, REPORT,      REPORT)       \
-  XX(17, MKACTIVITY,  MKACTIVITY)   \
+  XX(21, MKACTIVITY,  MKACTIVITY)   \
-  XX(18, CHECKOUT,    CHECKOUT)     \
+  XX(22, CHECKOUT,    CHECKOUT)     \
-  XX(19, MERGE,       MERGE)        \
+  XX(23, MERGE,       MERGE)        \
   /* upnp */                        \
-  XX(20, MSEARCH,     M-SEARCH)     \
+  XX(24, MSEARCH,     M-SEARCH)     \
-  XX(21, NOTIFY,      NOTIFY)       \
+  XX(25, NOTIFY,      NOTIFY)       \
-  XX(22, SUBSCRIBE,   SUBSCRIBE)    \
+  XX(26, SUBSCRIBE,   SUBSCRIBE)    \
-  XX(23, UNSUBSCRIBE, UNSUBSCRIBE)  \
+  XX(27, UNSUBSCRIBE, UNSUBSCRIBE)  \
   /* RFC-5789 */                    \
-  XX(24, PATCH,       PATCH)        \
+  XX(28, PATCH,       PATCH)        \
-  XX(25, PURGE,       PURGE)        \
+  XX(29, PURGE,       PURGE)        \
+  /* CalDAV */                      \
+  XX(30, MKCALENDAR,  MKCALENDAR)   \
+  /* RFC-2068, section 19.6.1.2 */  \
+  XX(31, LINK,        LINK)         \
+  XX(32, UNLINK,      UNLINK)       \
 
 enum http_method
   {
@@ -125,14 +218,16 @@ enum flags
   { F_CHUNKED               = 1 << 0
   , F_CONNECTION_KEEP_ALIVE = 1 << 1
   , F_CONNECTION_CLOSE      = 1 << 2
-  , F_TRAILING              = 1 << 3
+  , F_CONNECTION_UPGRADE    = 1 << 3
-  , F_UPGRADE               = 1 << 4
+  , F_TRAILING              = 1 << 4
-  , F_SKIPBODY              = 1 << 5
+  , F_UPGRADE               = 1 << 5
+  , F_SKIPBODY              = 1 << 6
+  , F_CONTENTLENGTH         = 1 << 7
   };
 
 
 /* Map for errno-related constants
- * 
+ *
  * The provided argument should be a macro that takes 2 arguments.
  */
 #define HTTP_ERRNO_MAP(XX)                                           \
@@ -141,13 +236,15 @@ enum flags
                                                                      \
   /* Callback-related errors */                                      \
   XX(CB_message_begin, "the on_message_begin callback failed")       \
-  XX(CB_status_complete, "the on_status_complete callback failed")   \
   XX(CB_url, "the on_url callback failed")                           \
   XX(CB_header_field, "the on_header_field callback failed")         \
   XX(CB_header_value, "the on_header_value callback failed")         \
   XX(CB_headers_complete, "the on_headers_complete callback failed") \
   XX(CB_body, "the on_body callback failed")                         \
   XX(CB_message_complete, "the on_message_complete callback failed") \
+  XX(CB_status, "the on_status callback failed")                     \
+  XX(CB_chunk_header, "the on_chunk_header callback failed")         \
+  XX(CB_chunk_complete, "the on_chunk_complete callback failed")     \
                                                                      \
   /* Parsing-related errors */                                       \
   XX(INVALID_EOF_STATE, "stream ended at an unexpected time")        \
@@ -168,6 +265,8 @@ enum flags
   XX(INVALID_HEADER_TOKEN, "invalid character in header")            \
   XX(INVALID_CONTENT_LENGTH,                                         \
      "invalid character in content-length header")                   \
+  XX(UNEXPECTED_CONTENT_LENGTH,                                      \
+     "unexpected content-length header")                             \
   XX(INVALID_CHUNK_SIZE,                                             \
      "invalid character in chunk size header")                       \
   XX(INVALID_CONSTANT, "invalid constant string")                    \
@@ -191,11 +290,12 @@ enum http_errno {
 
 struct http_parser {
   /** PRIVATE **/
-  unsigned char type : 2;     /* enum http_parser_type */
+  unsigned int type : 2;         /* enum http_parser_type */
-  unsigned char flags : 6;    /* F_* values from 'flags' enum; semi-public */
+  unsigned int flags : 8;        /* F_* values from 'flags' enum; semi-public */
-  unsigned char state;        /* enum state from http_parser.c */
+  unsigned int state : 7;        /* enum state from http_parser.c */
-  unsigned char header_state; /* enum header_state from http_parser.c */
+  unsigned int header_state : 7; /* enum header_state from http_parser.c */
-  unsigned char index;        /* index into current matcher */
+  unsigned int index : 7;        /* index into current matcher */
+  unsigned int lenient_http_headers : 1;
 
   uint32_t nread;          /* # bytes read in various scenarios */
   uint64_t content_length; /* # bytes in body (0 if no Content-Length header) */
@@ -203,16 +303,16 @@ struct http_parser {
   /** READ-ONLY **/
   unsigned short http_major;
   unsigned short http_minor;
-  unsigned short status_code; /* responses only */
+  unsigned int status_code : 16; /* responses only */
-  unsigned char method;       /* requests only */
+  unsigned int method : 8;       /* requests only */
-  unsigned char http_errno : 7;
+  unsigned int http_errno : 7;
 
   /* 1 = Upgrade header was present and the parser has exited because of that.
    * 0 = No upgrade header present.
    * Should be checked when http_parser_execute() returns in addition to
    * error checking.
    */
-  unsigned char upgrade : 1;
+  unsigned int upgrade : 1;
 
   /** PUBLIC **/
   void *data; /* A pointer to get hook to the "connection" or "socket" object */
@@ -222,12 +322,17 @@ struct http_parser {
 struct http_parser_settings {
   http_cb      on_message_begin;
   http_data_cb on_url;
-  http_cb      on_status_complete;
+  http_data_cb on_status;
   http_data_cb on_header_field;
   http_data_cb on_header_value;
   http_cb      on_headers_complete;
   http_data_cb on_body;
   http_cb      on_message_complete;
+  /* When on_chunk_header is called, the current chunk length is stored
+   * in parser->content_length.
+   */
+  http_cb      on_chunk_header;
+  http_cb      on_chunk_complete;
 };
 
 
@@ -261,9 +366,28 @@ struct http_parser_url {
 };
 
 
+/* Returns the library version. Bits 16-23 contain the major version number,
+ * bits 8-15 the minor version number and bits 0-7 the patch level.
+ * Usage example:
+ *
+ *   unsigned long version = http_parser_version();
+ *   unsigned major = (version >> 16) & 255;
+ *   unsigned minor = (version >> 8) & 255;
+ *   unsigned patch = version & 255;
+ *   printf("http_parser v%u.%u.%u\n", major, minor, patch);
+ */
+unsigned long http_parser_version(void);
+
 void http_parser_init(http_parser *parser, enum http_parser_type type);
 
 
+/* Initialize http_parser_settings members to 0
+ */
+void http_parser_settings_init(http_parser_settings *settings);
+
+
+/* Executes the parser. Returns number of parsed bytes. Sets
+ * `parser->http_errno` on error. */
 size_t http_parser_execute(http_parser *parser,
                            const http_parser_settings *settings,
                            const char *data,
@@ -287,6 +411,9 @@ const char *http_errno_name(enum http_errno err);
 /* Return a string description of the given error */
 const char *http_errno_description(enum http_errno err);
 
+/* Initialize all http_parser_url members to 0 */
+void http_parser_url_init(struct http_parser_url *u);
+
 /* Parse a URL; return nonzero on failure */
 int http_parser_parse_url(const char *buf, size_t buflen,
                           int is_connect,

src/main.h

@@ -0,0 +1,54 @@
+#ifndef MAIN_H
+#define MAIN_H
+
+#include <ctype.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <getopt.h>
+#include <math.h>
+#include <netinet/in.h>
+#include <netinet/tcp.h>
+#include <stdarg.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <signal.h>
+#include <time.h>
+#include <unistd.h>
+#include <sys/time.h>
+#include <sys/uio.h>
+
+#include "ssl.h"
+#include "aprintf.h"
+#include "stats.h"
+#include "units.h"
+#include "zmalloc.h"
+
+struct config;
+
+static void *thread_main(void *);
+static int connect_socket(thread *, connection *);
+static int reconnect_socket(thread *, connection *);
+
+static int record_rate(aeEventLoop *, long long, void *);
+
+static void socket_connected(aeEventLoop *, int, void *, int);
+static void socket_writeable(aeEventLoop *, int, void *, int);
+static void socket_readable(aeEventLoop *, int, void *, int);
+
+static int response_complete(http_parser *);
+static int header_field(http_parser *, const char *, size_t);
+static int header_value(http_parser *, const char *, size_t);
+static int response_body(http_parser *, const char *, size_t);
+
+static uint64_t time_us();
+
+static int parse_args(struct config *, char **, struct http_parser_url *, char **, int, char **);
+static char *copy_url_part(char *, struct http_parser_url *, enum http_parser_url_fields);
+
+static void print_stats_header();
+static void print_stats(char *, stats *, char *(*)(long double));
+static void print_stats_latency(stats *);
+
+#endif /* MAIN_H */

src/net.c

@@ -0,0 +1,39 @@
+// Copyright (C) 2013 - Will Glozer.  All rights reserved.
+
+#include <errno.h>
+#include <unistd.h>
+#include <sys/ioctl.h>
+
+#include "net.h"
+
+status sock_connect(connection *c, char *host) {
+    return OK;
+}
+
+status sock_close(connection *c) {
+    return OK;
+}
+
+status sock_read(connection *c, size_t *n) {
+    ssize_t r = read(c->fd, c->buf, sizeof(c->buf));
+    *n = (size_t) r;
+    return r >= 0 ? OK : ERROR;
+}
+
+status sock_write(connection *c, char *buf, size_t len, size_t *n) {
+    ssize_t r;
+    if ((r = write(c->fd, buf, len)) == -1) {
+        switch (errno) {
+            case EAGAIN: return RETRY;
+            default:     return ERROR;
+        }
+    }
+    *n = (size_t) r;
+    return OK;
+}
+
+size_t sock_readable(connection *c) {
+    int n, rc;
+    rc = ioctl(c->fd, FIONREAD, &n);
+    return rc == -1 ? 0 : n;
+}

src/net.h

@@ -0,0 +1,29 @@
+#ifndef NET_H
+#define NET_H
+
+#include "config.h"
+#include <stdint.h>
+#include <openssl/ssl.h>
+#include "wrk.h"
+
+typedef enum {
+    OK,
+    ERROR,
+    RETRY
+} status;
+
+struct sock {
+    status ( *connect)(connection *, char *);
+    status (   *close)(connection *);
+    status (    *read)(connection *, size_t *);
+    status (   *write)(connection *, char *, size_t, size_t *);
+    size_t (*readable)(connection *);
+};
+
+status sock_connect(connection *, char *);
+status sock_close(connection *);
+status sock_read(connection *, size_t *);
+status sock_write(connection *, char *, size_t, size_t *);
+size_t sock_readable(connection *);
+
+#endif /* NET_H */

src/script.c

@@ -0,0 +1,576 @@
+// Copyright (C) 2013 - Will Glozer.  All rights reserved.
+
+#include <stdlib.h>
+#include <string.h>
+#include "script.h"
+#include "http_parser.h"
+#include "zmalloc.h"
+
+typedef struct {
+    char *name;
+    int   type;
+    void *value;
+} table_field;
+
+static int script_addr_tostring(lua_State *);
+static int script_addr_gc(lua_State *);
+static int script_stats_call(lua_State *);
+static int script_stats_len(lua_State *);
+static int script_stats_index(lua_State *);
+static int script_thread_index(lua_State *);
+static int script_thread_newindex(lua_State *);
+static int script_wrk_lookup(lua_State *);
+static int script_wrk_connect(lua_State *);
+
+static void set_fields(lua_State *, int, const table_field *);
+static void set_field(lua_State *, int, char *, int);
+static int push_url_part(lua_State *, char *, struct http_parser_url *, enum http_parser_url_fields);
+
+static const struct luaL_Reg addrlib[] = {
+    { "__tostring", script_addr_tostring   },
+    { "__gc"    ,   script_addr_gc         },
+    { NULL,         NULL                   }
+};
+
+static const struct luaL_Reg statslib[] = {
+    { "__call",     script_stats_call      },
+    { "__index",    script_stats_index     },
+    { "__len",      script_stats_len       },
+    { NULL,         NULL                   }
+};
+
+static const struct luaL_Reg threadlib[] = {
+    { "__index",    script_thread_index    },
+    { "__newindex", script_thread_newindex },
+    { NULL,         NULL                   }
+};
+
+lua_State *script_create(char *file, char *url, char **headers) {
+    lua_State *L = luaL_newstate();
+    luaL_openlibs(L);
+    (void) luaL_dostring(L, "wrk = require \"wrk\"");
+
+    luaL_newmetatable(L, "wrk.addr");
+    luaL_register(L, NULL, addrlib);
+    luaL_newmetatable(L, "wrk.stats");
+    luaL_register(L, NULL, statslib);
+    luaL_newmetatable(L, "wrk.thread");
+    luaL_register(L, NULL, threadlib);
+
+    struct http_parser_url parts = {};
+    script_parse_url(url, &parts);
+    char *path = "/";
+
+    if (parts.field_set & (1 << UF_PATH)) {
+        path = &url[parts.field_data[UF_PATH].off];
+    }
+
+    const table_field fields[] = {
+        { "lookup",  LUA_TFUNCTION, script_wrk_lookup  },
+        { "connect", LUA_TFUNCTION, script_wrk_connect },
+        { "path",    LUA_TSTRING,   path               },
+        { NULL,      0,             NULL               },
+    };
+
+    lua_getglobal(L, "wrk");
+
+    set_field(L, 4, "scheme", push_url_part(L, url, &parts, UF_SCHEMA));
+    set_field(L, 4, "host",   push_url_part(L, url, &parts, UF_HOST));
+    set_field(L, 4, "port",   push_url_part(L, url, &parts, UF_PORT));
+    set_fields(L, 4, fields);
+
+    lua_getfield(L, 4, "headers");
+    for (char **h = headers; *h; h++) {
+        char *p = strchr(*h, ':');
+        if (p && p[1] == ' ') {
+            lua_pushlstring(L, *h, p - *h);
+            lua_pushstring(L, p + 2);
+            lua_settable(L, 5);
+        }
+    }
+    lua_pop(L, 5);
+
+    if (file && luaL_dofile(L, file)) {
+        const char *cause = lua_tostring(L, -1);
+        fprintf(stderr, "%s: %s\n", file, cause);
+    }
+
+    return L;
+}
+
+bool script_resolve(lua_State *L, char *host, char *service) {
+    lua_getglobal(L, "wrk");
+
+    lua_getfield(L, -1, "resolve");
+    lua_pushstring(L, host);
+    lua_pushstring(L, service);
+    lua_call(L, 2, 0);
+
+    lua_getfield(L, -1, "addrs");
+    size_t count = lua_objlen(L, -1);
+    lua_pop(L, 2);
+    return count > 0;
+}
+
+void script_push_thread(lua_State *L, thread *t) {
+    thread **ptr = (thread **) lua_newuserdata(L, sizeof(thread **));
+    *ptr = t;
+    luaL_getmetatable(L, "wrk.thread");
+    lua_setmetatable(L, -2);
+}
+
+void script_init(lua_State *L, thread *t, int argc, char **argv) {
+    lua_getglobal(t->L, "wrk");
+
+    script_push_thread(t->L, t);
+    lua_setfield(t->L, -2, "thread");
+
+    lua_getglobal(L, "wrk");
+    lua_getfield(L, -1, "setup");
+    script_push_thread(L, t);
+    lua_call(L, 1, 0);
+    lua_pop(L, 1);
+
+    lua_getfield(t->L, -1, "init");
+    lua_newtable(t->L);
+    for (int i = 0; i < argc; i++) {
+        lua_pushstring(t->L, argv[i]);
+        lua_rawseti(t->L, -2, i);
+    }
+    lua_call(t->L, 1, 0);
+    lua_pop(t->L, 1);
+}
+
+uint64_t script_delay(lua_State *L) {
+    lua_getglobal(L, "delay");
+    lua_call(L, 0, 1);
+    uint64_t delay = lua_tonumber(L, -1);
+    lua_pop(L, 1);
+    return delay;
+}
+
+void script_request(lua_State *L, char **buf, size_t *len) {
+    int pop = 1;
+    lua_getglobal(L, "request");
+    if (!lua_isfunction(L, -1)) {
+        lua_getglobal(L, "wrk");
+        lua_getfield(L, -1, "request");
+        pop += 2;
+    }
+    lua_call(L, 0, 1);
+    const char *str = lua_tolstring(L, -1, len);
+    *buf = realloc(*buf, *len);
+    memcpy(*buf, str, *len);
+    lua_pop(L, pop);
+}
+
+void script_response(lua_State *L, int status, buffer *headers, buffer *body) {
+    lua_getglobal(L, "response");
+    lua_pushinteger(L, status);
+    lua_newtable(L);
+
+    for (char *c = headers->buffer; c < headers->cursor; ) {
+        c = buffer_pushlstring(L, c);
+        c = buffer_pushlstring(L, c);
+        lua_rawset(L, -3);
+    }
+
+    lua_pushlstring(L, body->buffer, body->cursor - body->buffer);
+    lua_call(L, 3, 0);
+
+    buffer_reset(headers);
+    buffer_reset(body);
+}
+
+bool script_is_function(lua_State *L, char *name) {
+    lua_getglobal(L, name);
+    bool is_function = lua_isfunction(L, -1);
+    lua_pop(L, 1);
+    return is_function;
+}
+
+bool script_is_static(lua_State *L) {
+    return !script_is_function(L, "request");
+}
+
+bool script_want_response(lua_State *L) {
+    return script_is_function(L, "response");
+}
+
+bool script_has_delay(lua_State *L) {
+    return script_is_function(L, "delay");
+}
+
+bool script_has_done(lua_State *L) {
+    return script_is_function(L, "done");
+}
+
+void script_header_done(lua_State *L, luaL_Buffer *buffer) {
+    luaL_pushresult(buffer);
+}
+
+void script_summary(lua_State *L, uint64_t duration, uint64_t requests, uint64_t bytes) {
+    const table_field fields[] = {
+        { "duration", LUA_TNUMBER, &duration },
+        { "requests", LUA_TNUMBER, &requests },
+        { "bytes",    LUA_TNUMBER, &bytes    },
+        { NULL,       0,           NULL      },
+    };
+    lua_newtable(L);
+    set_fields(L, 1, fields);
+}
+
+void script_errors(lua_State *L, errors *errors) {
+    uint64_t e[] = {
+        errors->connect,
+        errors->read,
+        errors->write,
+        errors->status,
+        errors->timeout
+    };
+    const table_field fields[] = {
+        { "connect", LUA_TNUMBER, &e[0] },
+        { "read",    LUA_TNUMBER, &e[1] },
+        { "write",   LUA_TNUMBER, &e[2] },
+        { "status",  LUA_TNUMBER, &e[3] },
+        { "timeout", LUA_TNUMBER, &e[4] },
+        { NULL,      0,           NULL  },
+    };
+    lua_newtable(L);
+    set_fields(L, 2, fields);
+    lua_setfield(L, 1, "errors");
+}
+
+void script_push_stats(lua_State *L, stats *s) {
+    stats **ptr = (stats **) lua_newuserdata(L, sizeof(stats **));
+    *ptr = s;
+    luaL_getmetatable(L, "wrk.stats");
+    lua_setmetatable(L, -2);
+}
+
+void script_done(lua_State *L, stats *latency, stats *requests) {
+    lua_getglobal(L, "done");
+    lua_pushvalue(L, 1);
+
+    script_push_stats(L, latency);
+    script_push_stats(L, requests);
+
+    lua_call(L, 3, 0);
+    lua_pop(L, 1);
+}
+
+static int verify_request(http_parser *parser) {
+    size_t *count = parser->data;
+    (*count)++;
+    return 0;
+}
+
+size_t script_verify_request(lua_State *L) {
+    http_parser_settings settings = {
+        .on_message_complete = verify_request
+    };
+    http_parser parser;
+    char *request = NULL;
+    size_t len, count = 0;
+
+    script_request(L, &request, &len);
+    http_parser_init(&parser, HTTP_REQUEST);
+    parser.data = &count;
+
+    size_t parsed = http_parser_execute(&parser, &settings, request, len);
+
+    if (parsed != len || count == 0) {
+        enum http_errno err = HTTP_PARSER_ERRNO(&parser);
+        const char *desc = http_errno_description(err);
+        const char *msg = err != HPE_OK ? desc : "incomplete request";
+        int line = 1, column = 1;
+
+        for (char *c = request; c < request + parsed; c++) {
+            column++;
+            if (*c == '\n') {
+                column = 1;
+                line++;
+            }
+        }
+
+        fprintf(stderr, "%s at %d:%d\n", msg, line, column);
+        exit(1);
+    }
+
+    return count;
+}
+
+static struct addrinfo *checkaddr(lua_State *L) {
+    struct addrinfo *addr = luaL_checkudata(L, -1, "wrk.addr");
+    luaL_argcheck(L, addr != NULL, 1, "`addr' expected");
+    return addr;
+}
+
+void script_addr_copy(struct addrinfo *src, struct addrinfo *dst) {
+    *dst = *src;
+    dst->ai_addr = zmalloc(src->ai_addrlen);
+    memcpy(dst->ai_addr, src->ai_addr, src->ai_addrlen);
+}
+
+struct addrinfo *script_addr_clone(lua_State *L, struct addrinfo *addr) {
+    struct addrinfo *udata = lua_newuserdata(L, sizeof(*udata));
+    luaL_getmetatable(L, "wrk.addr");
+    lua_setmetatable(L, -2);
+    script_addr_copy(addr, udata);
+    return udata;
+}
+
+static int script_addr_tostring(lua_State *L) {
+    struct addrinfo *addr = checkaddr(L);
+    char host[NI_MAXHOST];
+    char service[NI_MAXSERV];
+
+    int flags = NI_NUMERICHOST | NI_NUMERICSERV;
+    int rc = getnameinfo(addr->ai_addr, addr->ai_addrlen, host, NI_MAXHOST, service, NI_MAXSERV, flags);
+    if (rc != 0) {
+        const char *msg = gai_strerror(rc);
+        return luaL_error(L, "addr tostring failed %s", msg);
+    }
+
+    lua_pushfstring(L, "%s:%s", host, service);
+    return 1;
+}
+
+static int script_addr_gc(lua_State *L) {
+    struct addrinfo *addr = checkaddr(L);
+    zfree(addr->ai_addr);
+    return 0;
+}
+
+static stats *checkstats(lua_State *L) {
+    stats **s = luaL_checkudata(L, 1, "wrk.stats");
+    luaL_argcheck(L, s != NULL, 1, "`stats' expected");
+    return *s;
+}
+
+static int script_stats_percentile(lua_State *L) {
+    stats *s = checkstats(L);
+    lua_Number p = luaL_checknumber(L, 2);
+    lua_pushnumber(L, stats_percentile(s, p));
+    return 1;
+}
+
+static int script_stats_call(lua_State *L) {
+    stats *s = checkstats(L);
+    uint64_t index = lua_tonumber(L, 2);
+    uint64_t count;
+    lua_pushnumber(L, stats_value_at(s, index - 1, &count));
+    lua_pushnumber(L, count);
+    return 2;
+}
+
+static int script_stats_index(lua_State *L) {
+    stats *s = checkstats(L);
+    const char *method = lua_tostring(L, 2);
+    if (!strcmp("min",   method)) lua_pushnumber(L, s->min);
+    if (!strcmp("max",   method)) lua_pushnumber(L, s->max);
+    if (!strcmp("mean",  method)) lua_pushnumber(L, stats_mean(s));
+    if (!strcmp("stdev", method)) lua_pushnumber(L, stats_stdev(s, stats_mean(s)));
+    if (!strcmp("percentile", method)) {
+        lua_pushcfunction(L, script_stats_percentile);
+    }
+    return 1;
+}
+
+static int script_stats_len(lua_State *L) {
+    stats *s = checkstats(L);
+    lua_pushinteger(L, stats_popcount(s));
+    return 1;
+}
+
+static thread *checkthread(lua_State *L) {
+    thread **t = luaL_checkudata(L, 1, "wrk.thread");
+    luaL_argcheck(L, t != NULL, 1, "`thread' expected");
+    return *t;
+}
+
+static int script_thread_get(lua_State *L) {
+    thread *t = checkthread(L);
+    const char *key = lua_tostring(L, -1);
+    lua_getglobal(t->L, key);
+    script_copy_value(t->L, L, -1);
+    lua_pop(t->L, 1);
+    return 1;
+}
+
+static int script_thread_set(lua_State *L) {
+    thread *t = checkthread(L);
+    const char *name = lua_tostring(L, -2);
+    script_copy_value(L, t->L, -1);
+    lua_setglobal(t->L, name);
+    return 0;
+}
+
+static int script_thread_stop(lua_State *L) {
+    thread *t = checkthread(L);
+    aeStop(t->loop);
+    return 0;
+}
+
+static int script_thread_index(lua_State *L) {
+    thread *t = checkthread(L);
+    const char *key = lua_tostring(L, 2);
+    if (!strcmp("get",  key)) lua_pushcfunction(L, script_thread_get);
+    if (!strcmp("set",  key)) lua_pushcfunction(L, script_thread_set);
+    if (!strcmp("stop", key)) lua_pushcfunction(L, script_thread_stop);
+    if (!strcmp("addr", key)) script_addr_clone(L, t->addr);
+    return 1;
+}
+
+static int script_thread_newindex(lua_State *L) {
+    thread *t = checkthread(L);
+    const char *key = lua_tostring(L, -2);
+    if (!strcmp("addr", key)) {
+        struct addrinfo *addr = checkaddr(L);
+        if (t->addr) zfree(t->addr->ai_addr);
+        t->addr = zrealloc(t->addr, sizeof(*addr));
+        script_addr_copy(addr, t->addr);
+    } else {
+        luaL_error(L, "cannot set '%s' on thread", luaL_typename(L, -1));
+    }
+    return 0;
+}
+
+static int script_wrk_lookup(lua_State *L) {
+    struct addrinfo *addrs;
+    struct addrinfo hints = {
+        .ai_family   = AF_UNSPEC,
+        .ai_socktype = SOCK_STREAM
+    };
+    int rc, index = 1;
+
+    const char *host    = lua_tostring(L, -2);
+    const char *service = lua_tostring(L, -1);
+
+    if ((rc = getaddrinfo(host, service, &hints, &addrs)) != 0) {
+        const char *msg = gai_strerror(rc);
+        fprintf(stderr, "unable to resolve %s:%s %s\n", host, service, msg);
+        exit(1);
+    }
+
+    lua_newtable(L);
+    for (struct addrinfo *addr = addrs; addr != NULL; addr = addr->ai_next) {
+        script_addr_clone(L, addr);
+        lua_rawseti(L, -2, index++);
+    }
+
+    freeaddrinfo(addrs);
+    return 1;
+}
+
+static int script_wrk_connect(lua_State *L) {
+    struct addrinfo *addr = checkaddr(L);
+    int fd, connected = 0;
+    if ((fd = socket(addr->ai_family, addr->ai_socktype, addr->ai_protocol)) != -1) {
+        connected = connect(fd, addr->ai_addr, addr->ai_addrlen) == 0;
+        close(fd);
+    }
+    lua_pushboolean(L, connected);
+    return 1;
+}
+
+void script_copy_value(lua_State *src, lua_State *dst, int index) {
+    switch (lua_type(src, index)) {
+        case LUA_TBOOLEAN:
+            lua_pushboolean(dst, lua_toboolean(src, index));
+            break;
+        case LUA_TNIL:
+            lua_pushnil(dst);
+            break;
+        case LUA_TNUMBER:
+            lua_pushnumber(dst, lua_tonumber(src, index));
+            break;
+        case LUA_TSTRING:
+            lua_pushstring(dst, lua_tostring(src, index));
+            break;
+        case LUA_TTABLE:
+            lua_newtable(dst);
+            lua_pushnil(src);
+            while (lua_next(src, index - 1)) {
+                script_copy_value(src, dst, -2);
+                script_copy_value(src, dst, -1);
+                lua_settable(dst, -3);
+                lua_pop(src, 1);
+            }
+            lua_pop(src, 1);
+            break;
+        default:
+            luaL_error(src, "cannot transfer '%s' to thread", luaL_typename(src, index));
+    }
+}
+
+int script_parse_url(char *url, struct http_parser_url *parts) {
+    if (!http_parser_parse_url(url, strlen(url), 0, parts)) {
+        if (!(parts->field_set & (1 << UF_SCHEMA))) return 0;
+        if (!(parts->field_set & (1 << UF_HOST)))   return 0;
+        return 1;
+    }
+    return 0;
+}
+
+static int push_url_part(lua_State *L, char *url, struct http_parser_url *parts, enum http_parser_url_fields field) {
+    int type = parts->field_set & (1 << field) ? LUA_TSTRING : LUA_TNIL;
+    uint16_t off, len;
+    switch (type) {
+        case LUA_TSTRING:
+            off = parts->field_data[field].off;
+            len = parts->field_data[field].len;
+            lua_pushlstring(L, &url[off], len);
+            break;
+        case LUA_TNIL:
+            lua_pushnil(L);
+    }
+    return type;
+}
+
+static void set_field(lua_State *L, int index, char *field, int type) {
+    (void) type;
+    lua_setfield(L, index, field);
+}
+
+static void set_fields(lua_State *L, int index, const table_field *fields) {
+    for (int i = 0; fields[i].name; i++) {
+        table_field f = fields[i];
+        switch (f.value == NULL ? LUA_TNIL : f.type) {
+            case LUA_TFUNCTION:
+                lua_pushcfunction(L, (lua_CFunction) f.value);
+                break;
+            case LUA_TNUMBER:
+                lua_pushinteger(L, *((lua_Integer *) f.value));
+                break;
+            case LUA_TSTRING:
+                lua_pushstring(L, (const char *) f.value);
+                break;
+            case LUA_TNIL:
+                lua_pushnil(L);
+                break;
+        }
+        lua_setfield(L, index, f.name);
+    }
+}
+
+void buffer_append(buffer *b, const char *data, size_t len) {
+    size_t used = b->cursor - b->buffer;
+    while (used + len + 1 >= b->length) {
+        b->length += 1024;
+        b->buffer  = realloc(b->buffer, b->length);
+        b->cursor  = b->buffer + used;
+    }
+    memcpy(b->cursor, data, len);
+    b->cursor += len;
+}
+
+void buffer_reset(buffer *b) {
+    b->cursor = b->buffer;
+}
+
+char *buffer_pushlstring(lua_State *L, char *start) {
+    char *end = strchr(start, 0);
+    lua_pushlstring(L, start, end - start);
+    return end + 1;
+}

src/script.h

@@ -0,0 +1,38 @@
+#ifndef SCRIPT_H
+#define SCRIPT_H
+
+#include <stdbool.h>
+#include <lua.h>
+#include <lualib.h>
+#include <lauxlib.h>
+#include <unistd.h>
+#include "stats.h"
+#include "wrk.h"
+
+lua_State *script_create(char *, char *, char **);
+
+bool script_resolve(lua_State *, char *, char *);
+void script_setup(lua_State *, thread *);
+void script_done(lua_State *, stats *, stats *);
+
+void script_init(lua_State *, thread *, int, char **);
+uint64_t script_delay(lua_State *);
+void script_request(lua_State *, char **, size_t *);
+void script_response(lua_State *, int, buffer *, buffer *);
+size_t script_verify_request(lua_State *L);
+
+bool script_is_static(lua_State *);
+bool script_want_response(lua_State *L);
+bool script_has_delay(lua_State *L);
+bool script_has_done(lua_State *L);
+void script_summary(lua_State *, uint64_t, uint64_t, uint64_t);
+void script_errors(lua_State *, errors *);
+
+void script_copy_value(lua_State *, lua_State *, int);
+int script_parse_url(char *, struct http_parser_url *);
+
+void buffer_append(buffer *, const char *, size_t);
+void buffer_reset(buffer *);
+char *buffer_pushlstring(lua_State *, char *);
+
+#endif /* SCRIPT_H */

src/ssl.c

@@ -0,0 +1,76 @@
+// Copyright (C) 2013 - Will Glozer.  All rights reserved.
+
+#include <pthread.h>
+
+#include <openssl/evp.h>
+#include <openssl/err.h>
+#include <openssl/ssl.h>
+
+#include "ssl.h"
+
+SSL_CTX *ssl_init() {
+    SSL_CTX *ctx = NULL;
+
+    SSL_load_error_strings();
+    SSL_library_init();
+    OpenSSL_add_all_algorithms();
+
+    if ((ctx = SSL_CTX_new(SSLv23_client_method()))) {
+        SSL_CTX_set_verify(ctx, SSL_VERIFY_NONE, NULL);
+        SSL_CTX_set_verify_depth(ctx, 0);
+        SSL_CTX_set_mode(ctx, SSL_MODE_AUTO_RETRY);
+        SSL_CTX_set_session_cache_mode(ctx, SSL_SESS_CACHE_CLIENT);
+    }
+
+    return ctx;
+}
+
+status ssl_connect(connection *c, char *host) {
+    int r;
+    SSL_set_fd(c->ssl, c->fd);
+    SSL_set_tlsext_host_name(c->ssl, host);
+    if ((r = SSL_connect(c->ssl)) != 1) {
+        switch (SSL_get_error(c->ssl, r)) {
+            case SSL_ERROR_WANT_READ:  return RETRY;
+            case SSL_ERROR_WANT_WRITE: return RETRY;
+            default:                   return ERROR;
+        }
+    }
+    return OK;
+}
+
+status ssl_close(connection *c) {
+    SSL_shutdown(c->ssl);
+    SSL_clear(c->ssl);
+    return OK;
+}
+
+status ssl_read(connection *c, size_t *n) {
+    int r;
+    if ((r = SSL_read(c->ssl, c->buf, sizeof(c->buf))) <= 0) {
+        switch (SSL_get_error(c->ssl, r)) {
+            case SSL_ERROR_WANT_READ:  return RETRY;
+            case SSL_ERROR_WANT_WRITE: return RETRY;
+            default:                   return ERROR;
+        }
+    }
+    *n = (size_t) r;
+    return OK;
+}
+
+status ssl_write(connection *c, char *buf, size_t len, size_t *n) {
+    int r;
+    if ((r = SSL_write(c->ssl, buf, len)) <= 0) {
+        switch (SSL_get_error(c->ssl, r)) {
+            case SSL_ERROR_WANT_READ:  return RETRY;
+            case SSL_ERROR_WANT_WRITE: return RETRY;
+            default:                   return ERROR;
+        }
+    }
+    *n = (size_t) r;
+    return OK;
+}
+
+size_t ssl_readable(connection *c) {
+    return SSL_pending(c->ssl);
+}

src/ssl.h

@@ -0,0 +1,14 @@
+#ifndef SSL_H
+#define SSL_H
+
+#include "net.h"
+
+SSL_CTX *ssl_init();
+
+status ssl_connect(connection *, char *);
+status ssl_close(connection *);
+status ssl_read(connection *, size_t *);
+status ssl_write(connection *, char *, size_t, size_t *);
+size_t ssl_readable(connection *);
+
+#endif /* SSL_H */

src/stats.c

@@ -7,56 +7,60 @@
 #include "stats.h"
 #include "zmalloc.h"
 
-stats *stats_alloc(uint64_t samples) {
+stats *stats_alloc(uint64_t max) {
-    stats *stats = zcalloc(sizeof(stats) + sizeof(uint64_t) * samples);
+    uint64_t limit = max + 1;
-    stats->samples = samples;
+    stats *s = zcalloc(sizeof(stats) + sizeof(uint64_t) * limit);
-    return stats;
+    s->limit = limit;
+    s->min   = UINT64_MAX;
+    return s;
 }
 
 void stats_free(stats *stats) {
     zfree(stats);
 }
 
-void stats_record(stats *stats, uint64_t x) {
+int stats_record(stats *stats, uint64_t n) {
-    stats->data[stats->index++] = x;
+    if (n >= stats->limit) return 0;
-    if (stats->limit < stats->samples)  stats->limit++;
+    __sync_fetch_and_add(&stats->data[n], 1);
-    if (stats->index == stats->samples) stats->index = 0;
+    __sync_fetch_and_add(&stats->count, 1);
+    uint64_t min = stats->min;
+    uint64_t max = stats->max;
+    while (n < min) min = __sync_val_compare_and_swap(&stats->min, min, n);
+    while (n > max) max = __sync_val_compare_and_swap(&stats->max, max, n);
+    return 1;
 }
 
-uint64_t stats_min(stats *stats) {
+void stats_correct(stats *stats, int64_t expected) {
-    uint64_t min = 0;
+    for (uint64_t n = expected * 2; n <= stats->max; n++) {
-    for (uint64_t i = 0; i < stats->limit; i++) {
+        uint64_t count = stats->data[n];
-        uint64_t x = stats->data[i];
+        int64_t m = (int64_t) n - expected;
-        if (x < min || min == 0) min = x;
+        while (count && m > expected) {
+            stats->data[m] += count;
+            stats->count += count;
+            m -= expected;
+        }
     }
-    return min;
-}
-
-uint64_t stats_max(stats *stats) {
-    uint64_t max = 0;
-    for (uint64_t i = 0; i < stats->limit; i++) {
-        uint64_t x = stats->data[i];
-        if (x > max || max == 0) max = x;
-    }
-    return max;
 }
 
 long double stats_mean(stats *stats) {
+    if (stats->count == 0) return 0.0;
+
     uint64_t sum = 0;
-    if (stats->limit == 0) return 0.0;
+    for (uint64_t i = stats->min; i <= stats->max; i++) {
-    for (uint64_t i = 0; i < stats->limit; i++) {
+        sum += stats->data[i] * i;
-        sum += stats->data[i];
     }
-    return sum / (long double) stats->limit;
+    return sum / (long double) stats->count;
 }
 
 long double stats_stdev(stats *stats, long double mean) {
     long double sum = 0.0;
-    if (stats->limit < 2) return 0.0;
+    if (stats->count < 2) return 0.0;
-    for (uint64_t i = 0; i < stats->limit; i++) {
+    for (uint64_t i = stats->min; i <= stats->max; i++) {
-        sum += powl(stats->data[i] - mean, 2);
+        if (stats->data[i]) {
+            sum += powl(i - mean, 2) * stats->data[i];
+        }
     }
-    return sqrtl(sum / (stats->limit - 1));
+    return sqrtl(sum / (stats->count - 1));
 }
 
 long double stats_within_stdev(stats *stats, long double mean, long double stdev, uint64_t n) {
@@ -64,10 +68,40 @@ long double stats_within_stdev(stats *stats, long double mean, long double stdev
     long double lower = mean - (stdev * n);
     uint64_t sum = 0;
 
-    for (uint64_t i = 0; i < stats->limit; i++) {
+    for (uint64_t i = stats->min; i <= stats->max; i++) {
-        uint64_t x = stats->data[i];
+        if (i >= lower && i <= upper) {
-        if (x >= lower && x <= upper) sum++;
+            sum += stats->data[i];
+        }
     }
 
-    return (sum / (long double) stats->limit) * 100;
+    return (sum / (long double) stats->count) * 100;
+}
+
+uint64_t stats_percentile(stats *stats, long double p) {
+    uint64_t rank = round((p / 100.0) * stats->count + 0.5);
+    uint64_t total = 0;
+    for (uint64_t i = stats->min; i <= stats->max; i++) {
+        total += stats->data[i];
+        if (total >= rank) return i;
+    }
+    return 0;
+}
+
+uint64_t stats_popcount(stats *stats) {
+    uint64_t count = 0;
+    for (uint64_t i = stats->min; i <= stats->max; i++) {
+        if (stats->data[i]) count++;
+    }
+    return count;
+}
+
+uint64_t stats_value_at(stats *stats, uint64_t index, uint64_t *count) {
+    *count = 0;
+    for (uint64_t i = stats->min; i <= stats->max; i++) {
+        if (stats->data[i] && (*count)++ == index) {
+            *count = stats->data[i];
+            return i;
+        }
+    }
+    return 0;
 }

src/stats.h

@@ -1,21 +1,40 @@
 #ifndef STATS_H
 #define STATS_H
 
+#include <stdbool.h>
+#include <stdint.h>
+
+#define MAX(X, Y) ((X) > (Y) ? (X) : (Y))
+#define MIN(X, Y) ((X) < (Y) ? (X) : (Y))
+
 typedef struct {
-    uint64_t samples;
+    uint32_t connect;
-    uint64_t index;
+    uint32_t read;
+    uint32_t write;
+    uint32_t status;
+    uint32_t timeout;
+} errors;
+
+typedef struct {
+    uint64_t count;
     uint64_t limit;
+    uint64_t min;
+    uint64_t max;
     uint64_t data[];
 } stats;
 
 stats *stats_alloc(uint64_t);
 void stats_free(stats *);
-void stats_record(stats *, uint64_t);
+
-uint64_t stats_min(stats *);
+int stats_record(stats *, uint64_t);
-uint64_t stats_max(stats *);
+void stats_correct(stats *, int64_t);
+
 long double stats_mean(stats *);
 long double stats_stdev(stats *stats, long double);
 long double stats_within_stdev(stats *, long double, long double, uint64_t);
+uint64_t stats_percentile(stats *, long double);
+
+uint64_t stats_popcount(stats *);
+uint64_t stats_value_at(stats *stats, uint64_t, uint64_t *);
 
 #endif /* STATS_H */
-

src/tinymt64.c

@@ -1,129 +0,0 @@
-/**
- * @file tinymt64.c
- *
- * @brief 64-bit Tiny Mersenne Twister only 127 bit internal state
- *
- * @author Mutsuo Saito (Hiroshima University)
- * @author Makoto Matsumoto (The University of Tokyo)
- *
- * Copyright (C) 2011 Mutsuo Saito, Makoto Matsumoto,
- * Hiroshima University and The University of Tokyo.
- * All rights reserved.
- *
- * The 3-clause BSD License is applied to this software, see
- * LICENSE.txt
- */
-#include "tinymt64.h"
-
-#define MIN_LOOP 8
-
-/**
- * This function represents a function used in the initialization
- * by init_by_array
- * @param[in] x 64-bit integer
- * @return 64-bit integer
- */
-static uint64_t ini_func1(uint64_t x) {
-    return (x ^ (x >> 59)) * UINT64_C(2173292883993);
-}
-
-/**
- * This function represents a function used in the initialization
- * by init_by_array
- * @param[in] x 64-bit integer
- * @return 64-bit integer
- */
-static uint64_t ini_func2(uint64_t x) {
-    return (x ^ (x >> 59)) * UINT64_C(58885565329898161);
-}
-
-/**
- * This function certificate the period of 2^127-1.
- * @param random tinymt state vector.
- */
-static void period_certification(tinymt64_t * random) {
-    if ((random->status[0] & TINYMT64_MASK) == 0 &&
-	random->status[1] == 0) {
-	random->status[0] = 'T';
-	random->status[1] = 'M';
-    }
-}
-
-/**
- * This function initializes the internal state array with a 64-bit
- * unsigned integer seed.
- * @param random tinymt state vector.
- * @param seed a 64-bit unsigned integer used as a seed.
- */
-void tinymt64_init(tinymt64_t * random, uint64_t seed) {
-    random->status[0] = seed ^ ((uint64_t)random->mat1 << 32);
-    random->status[1] = random->mat2 ^ random->tmat;
-    for (int i = 1; i < MIN_LOOP; i++) {
-	random->status[i & 1] ^= i + UINT64_C(6364136223846793005)
-	    * (random->status[(i - 1) & 1]
-	       ^ (random->status[(i - 1) & 1] >> 62));
-    }
-    period_certification(random);
-}
-
-/**
- * This function initializes the internal state array,
- * with an array of 64-bit unsigned integers used as seeds
- * @param random tinymt state vector.
- * @param init_key the array of 64-bit integers, used as a seed.
- * @param key_length the length of init_key.
- */
-void tinymt64_init_by_array(tinymt64_t * random, const uint64_t init_key[],
-			    int key_length) {
-    const int lag = 1;
-    const int mid = 1;
-    const int size = 4;
-    int i, j;
-    int count;
-    uint64_t r;
-    uint64_t st[4];
-
-    st[0] = 0;
-    st[1] = random->mat1;
-    st[2] = random->mat2;
-    st[3] = random->tmat;
-    if (key_length + 1 > MIN_LOOP) {
-	count = key_length + 1;
-    } else {
-	count = MIN_LOOP;
-    }
-    r = ini_func1(st[0] ^ st[mid % size]
-		  ^ st[(size - 1) % size]);
-    st[mid % size] += r;
-    r += key_length;
-    st[(mid + lag) % size] += r;
-    st[0] = r;
-    count--;
-    for (i = 1, j = 0; (j < count) && (j < key_length); j++) {
-	r = ini_func1(st[i] ^ st[(i + mid) % size] ^ st[(i + size - 1) % size]);
-	st[(i + mid) % size] += r;
-	r += init_key[j] + i;
-	st[(i + mid + lag) % size] += r;
-	st[i] = r;
-	i = (i + 1) % size;
-    }
-    for (; j < count; j++) {
-	r = ini_func1(st[i] ^ st[(i + mid) % size] ^ st[(i + size - 1) % size]);
-	st[(i + mid) % size] += r;
-	r += i;
-	st[(i + mid + lag) % size] += r;
-	st[i] = r;
-	i = (i + 1) % size;
-    }
-    for (j = 0; j < size; j++) {
-	r = ini_func2(st[i] + st[(i + mid) % size] + st[(i + size - 1) % size]);
-	st[(i + mid) % size] ^= r;
-	r -= i;
-	st[(i + mid + lag) % size] ^= r;
-	st[i] = r;
-	i = (i + 1) % size;
-    }
-    random->status[0] = st[0] ^ st[1];
-    random->status[1] = st[2] ^ st[3];
-    period_certification(random);
-}

src/tinymt64.h

@@ -1,210 +0,0 @@
-#ifndef TINYMT64_H
-#define TINYMT64_H
-/**
- * @file tinymt64.h
- *
- * @brief Tiny Mersenne Twister only 127 bit internal state
- *
- * @author Mutsuo Saito (Hiroshima University)
- * @author Makoto Matsumoto (The University of Tokyo)
- *
- * Copyright (C) 2011 Mutsuo Saito, Makoto Matsumoto,
- * Hiroshima University and The University of Tokyo.
- * All rights reserved.
- *
- * The 3-clause BSD License is applied to this software, see
- * LICENSE.txt
- */
-
-#include <stdint.h>
-#include <inttypes.h>
-
-#define TINYMT64_MEXP 127
-#define TINYMT64_SH0 12
-#define TINYMT64_SH1 11
-#define TINYMT64_SH8 8
-#define TINYMT64_MASK UINT64_C(0x7fffffffffffffff)
-#define TINYMT64_MUL (1.0 / 18446744073709551616.0)
-
-/*
- * tinymt64 internal state vector and parameters
- */
-struct TINYMT64_T {
-    uint64_t status[2];
-    uint32_t mat1;
-    uint32_t mat2;
-    uint64_t tmat;
-};
-
-typedef struct TINYMT64_T tinymt64_t;
-
-void tinymt64_init(tinymt64_t * random, uint64_t seed);
-void tinymt64_init_by_array(tinymt64_t * random, const uint64_t init_key[],
-			    int key_length);
-
-#if defined(__GNUC__)
-/**
- * This function always returns 127
- * @param random not used
- * @return always 127
- */
-inline static int tinymt64_get_mexp(
-    tinymt64_t * random  __attribute__((unused))) {
-    return TINYMT64_MEXP;
-}
-#else
-inline static int tinymt64_get_mexp(tinymt64_t * random) {
-    return TINYMT64_MEXP;
-}
-#endif
-
-/**
- * This function changes internal state of tinymt64.
- * Users should not call this function directly.
- * @param random tinymt internal status
- */
-inline static void tinymt64_next_state(tinymt64_t * random) {
-    uint64_t x;
-
-    random->status[0] &= TINYMT64_MASK;
-    x = random->status[0] ^ random->status[1];
-    x ^= x << TINYMT64_SH0;
-    x ^= x >> 32;
-    x ^= x << 32;
-    x ^= x << TINYMT64_SH1;
-    random->status[0] = random->status[1];
-    random->status[1] = x;
-    random->status[0] ^= -((int64_t)(x & 1)) & random->mat1;
-    random->status[1] ^= -((int64_t)(x & 1)) & (((uint64_t)random->mat2) << 32);
-}
-
-/**
- * This function outputs 64-bit unsigned integer from internal state.
- * Users should not call this function directly.
- * @param random tinymt internal status
- * @return 64-bit unsigned pseudorandom number
- */
-inline static uint64_t tinymt64_temper(tinymt64_t * random) {
-    uint64_t x;
-#if defined(LINEARITY_CHECK)
-    x = random->status[0] ^ random->status[1];
-#else
-    x = random->status[0] + random->status[1];
-#endif
-    x ^= random->status[0] >> TINYMT64_SH8;
-    x ^= -((int64_t)(x & 1)) & random->tmat;
-    return x;
-}
-
-/**
- * This function outputs floating point number from internal state.
- * Users should not call this function directly.
- * @param random tinymt internal status
- * @return floating point number r (1.0 <= r < 2.0)
- */
-inline static double tinymt64_temper_conv(tinymt64_t * random) {
-    uint64_t x;
-    union {
-	uint64_t u;
-	double d;
-    } conv;
-#if defined(LINEARITY_CHECK)
-    x = random->status[0] ^ random->status[1];
-#else
-    x = random->status[0] + random->status[1];
-#endif
-    x ^= random->status[0] >> TINYMT64_SH8;
-    conv.u = ((x ^ (-((int64_t)(x & 1)) & random->tmat)) >> 12)
-	| UINT64_C(0x3ff0000000000000);
-    return conv.d;
-}
-
-/**
- * This function outputs floating point number from internal state.
- * Users should not call this function directly.
- * @param random tinymt internal status
- * @return floating point number r (1.0 < r < 2.0)
- */
-inline static double tinymt64_temper_conv_open(tinymt64_t * random) {
-    uint64_t x;
-    union {
-	uint64_t u;
-	double d;
-    } conv;
-#if defined(LINEARITY_CHECK)
-    x = random->status[0] ^ random->status[1];
-#else
-    x = random->status[0] + random->status[1];
-#endif
-    x ^= random->status[0] >> TINYMT64_SH8;
-    conv.u = ((x ^ (-((int64_t)(x & 1)) & random->tmat)) >> 12)
-	| UINT64_C(0x3ff0000000000001);
-    return conv.d;
-}
-
-/**
- * This function outputs 64-bit unsigned integer from internal state.
- * @param random tinymt internal status
- * @return 64-bit unsigned integer r (0 <= r < 2^64)
- */
-inline static uint64_t tinymt64_generate_uint64(tinymt64_t * random) {
-    tinymt64_next_state(random);
-    return tinymt64_temper(random);
-}
-
-/**
- * This function outputs floating point number from internal state.
- * This function is implemented using multiplying by 1 / 2^64.
- * @param random tinymt internal status
- * @return floating point number r (0.0 <= r < 1.0)
- */
-inline static double tinymt64_generate_double(tinymt64_t * random) {
-    tinymt64_next_state(random);
-    return tinymt64_temper(random) * TINYMT64_MUL;
-}
-
-/**
- * This function outputs floating point number from internal state.
- * This function is implemented using union trick.
- * @param random tinymt internal status
- * @return floating point number r (0.0 <= r < 1.0)
- */
-inline static double tinymt64_generate_double01(tinymt64_t * random) {
-    tinymt64_next_state(random);
-    return tinymt64_temper_conv(random) - 1.0;
-}
-
-/**
- * This function outputs floating point number from internal state.
- * This function is implemented using union trick.
- * @param random tinymt internal status
- * @return floating point number r (1.0 <= r < 2.0)
- */
-inline static double tinymt64_generate_double12(tinymt64_t * random) {
-    tinymt64_next_state(random);
-    return tinymt64_temper_conv(random);
-}
-
-/**
- * This function outputs floating point number from internal state.
- * This function is implemented using union trick.
- * @param random tinymt internal status
- * @return floating point number r (0.0 < r <= 1.0)
- */
-inline static double tinymt64_generate_doubleOC(tinymt64_t * random) {
-    tinymt64_next_state(random);
-    return 2.0 - tinymt64_temper_conv(random);
-}
-
-/**
- * This function outputs floating point number from internal state.
- * This function is implemented using union trick.
- * @param random tinymt internal status
- * @return floating point number r (0.0 < r < 1.0)
- */
-inline static double tinymt64_generate_doubleOO(tinymt64_t * random) {
-    tinymt64_next_state(random);
-    return tinymt64_temper_conv_open(random) - 1.0;
-}
-
-#endif

src/wrk.c

@@ -1,53 +1,38 @@
 // Copyright (C) 2012 - Will Glozer.  All rights reserved.
 
 #include "wrk.h"
-
+#include "script.h"
-#include <ctype.h>
+#include "main.h"
-#include <errno.h>
-#include <fcntl.h>
-#include <getopt.h>
-#include <math.h>
-#include <netdb.h>
-#include <netinet/in.h>
-#include <netinet/tcp.h>
-#include <stdarg.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <signal.h>
-#include <time.h>
-#include <unistd.h>
-#include <sys/socket.h>
-#include <sys/time.h>
-#include <sys/uio.h>
-
-#include "aprintf.h"
-#include "stats.h"
-#include "units.h"
-#include "zmalloc.h"
-#include "tinymt64.h"
 
 static struct config {
-    struct addrinfo addr;
-    uint64_t threads;
     uint64_t connections;
     uint64_t duration;
+    uint64_t threads;
     uint64_t timeout;
+    uint64_t pipeline;
+    bool     delay;
+    bool     dynamic;
+    bool     latency;
+    char    *host;
+    char    *script;
+    SSL_CTX *ctx;
 } cfg;
 
-static struct {
-    size_t size;
-    char *buf;
-} request;
-
 static struct {
     stats *latency;
     stats *requests;
-    pthread_mutex_t mutex;
 } statistics;
 
-static const struct http_parser_settings parser_settings = {
+static struct sock sock = {
-    .on_message_complete = request_complete
+    .connect  = sock_connect,
+    .close    = sock_close,
+    .read     = sock_read,
+    .write    = sock_write,
+    .readable = sock_readable
+};
+
+static struct http_parser_settings parser_settings = {
+    .on_message_complete = response_complete
 };
 
 static volatile sig_atomic_t stop = 0;
@@ -63,7 +48,9 @@ static void usage() {
            "    -d, --duration    <T>  Duration of test           \n"
            "    -t, --threads     <N>  Number of threads to use   \n"
            "                                                      \n"
+           "    -s, --script      <S>  Load Lua script file       \n"
            "    -H, --header      <H>  Add header to request      \n"
+           "        --latency          Print latency statistics   \n"
            "        --timeout     <T>  Socket/request timeout     \n"
            "    -v, --version          Print version details      \n"
            "                                                      \n"
@@ -72,79 +59,70 @@ static void usage() {
 }
 
 int main(int argc, char **argv) {
-    struct addrinfo *addrs, *addr;
+    char *url, **headers = zmalloc(argc * sizeof(char *));
-    struct http_parser_url parser_url;
+    struct http_parser_url parts = {};
-    char *url, **headers;
-    int rc;
 
-    headers = zmalloc((argc / 2) * sizeof(char *));
+    if (parse_args(&cfg, &url, &parts, headers, argc, argv)) {
-
-    if (parse_args(&cfg, &url, headers, argc, argv)) {
         usage();
         exit(1);
     }
 
-    if (http_parser_parse_url(url, strlen(url), 0, &parser_url)) {
+    char *schema  = copy_url_part(url, &parts, UF_SCHEMA);
-        fprintf(stderr, "invalid URL: %s\n", url);
+    char *host    = copy_url_part(url, &parts, UF_HOST);
-        exit(1);
+    char *port    = copy_url_part(url, &parts, UF_PORT);
+    char *service = port ? port : schema;
+
+    if (!strncmp("https", schema, 5)) {
+        if ((cfg.ctx = ssl_init()) == NULL) {
+            fprintf(stderr, "unable to initialize SSL\n");
+            ERR_print_errors_fp(stderr);
+            exit(1);
+        }
+        sock.connect  = ssl_connect;
+        sock.close    = ssl_close;
+        sock.read     = ssl_read;
+        sock.write    = ssl_write;
+        sock.readable = ssl_readable;
     }
 
-    char *host = extract_url_part(url, &parser_url, UF_HOST);
+    signal(SIGPIPE, SIG_IGN);
-    char *port = extract_url_part(url, &parser_url, UF_PORT);
+    signal(SIGINT,  SIG_IGN);
-    char *service = port ? port : extract_url_part(url, &parser_url, UF_SCHEMA);
-    char *path = "/";
 
-    if (parser_url.field_set & (1 << UF_PATH)) {
+    statistics.latency  = stats_alloc(cfg.timeout * 1000);
-        path = &url[parser_url.field_data[UF_PATH].off];
+    statistics.requests = stats_alloc(MAX_THREAD_RATE_S);
-    }
+    thread *threads     = zcalloc(cfg.threads * sizeof(thread));
 
-    struct addrinfo hints = {
+    lua_State *L = script_create(cfg.script, url, headers);
-        .ai_family   = AF_UNSPEC,
+    if (!script_resolve(L, host, service)) {
-        .ai_socktype = SOCK_STREAM
-    };
-
-    if ((rc = getaddrinfo(host, service, &hints, &addrs)) != 0) {
-        const char *msg = gai_strerror(rc);
-        fprintf(stderr, "unable to resolve %s:%s %s\n", host, service, msg);
-        exit(1);
-    }
-
-    for (addr = addrs; addr != NULL; addr = addr->ai_next) {
-        int fd = socket(addr->ai_family, addr->ai_socktype, addr->ai_protocol);
-        if (fd == -1) continue;
-        rc = connect(fd, addr->ai_addr, addr->ai_addrlen);
-        close(fd);
-        if (rc == 0) break;
-    }
-
-    if (addr == NULL) {
         char *msg = strerror(errno);
         fprintf(stderr, "unable to connect to %s:%s %s\n", host, service, msg);
         exit(1);
     }
 
-    signal(SIGPIPE, SIG_IGN);
+    cfg.host = host;
-    signal(SIGINT,  SIG_IGN);
-    cfg.addr     = *addr;
-    request.buf  = format_request(host, port, path, headers);
-    request.size = strlen(request.buf);
-
-    pthread_mutex_init(&statistics.mutex, NULL);
-    statistics.latency  = stats_alloc(SAMPLES);
-    statistics.requests = stats_alloc(SAMPLES);
-
-    thread *threads = zcalloc(cfg.threads * sizeof(thread));
-    uint64_t connections = cfg.connections / cfg.threads;
-    uint64_t stop_at     = time_us() + (cfg.duration * 1000000);
 
     for (uint64_t i = 0; i < cfg.threads; i++) {
-        thread *t = &threads[i];
+        thread *t      = &threads[i];
-        t->connections = connections;
+        t->loop        = aeCreateEventLoop(10 + cfg.connections * 3);
-        t->stop_at     = stop_at;
+        t->connections = cfg.connections / cfg.threads;
 
-        if (pthread_create(&t->thread, NULL, &thread_main, t)) {
+        t->L = script_create(cfg.script, url, headers);
+        script_init(L, t, argc - optind, &argv[optind]);
+
+        if (i == 0) {
+            cfg.pipeline = script_verify_request(t->L);
+            cfg.dynamic  = !script_is_static(t->L);
+            cfg.delay    = script_has_delay(t->L);
+            if (script_want_response(t->L)) {
+                parser_settings.on_header_field = header_field;
+                parser_settings.on_header_value = header_value;
+                parser_settings.on_body         = response_body;
+            }
+        }
+
+        if (!t->loop || pthread_create(&t->thread, NULL, &thread_main, t)) {
             char *msg = strerror(errno);
-            fprintf(stderr, "unable to create thread %"PRIu64" %s\n", i, msg);
+            fprintf(stderr, "unable to create thread %"PRIu64": %s\n", i, msg);
             exit(2);
         }
     }
@@ -165,6 +143,9 @@ int main(int argc, char **argv) {
     uint64_t bytes    = 0;
     errors errors     = { 0 };
 
+    sleep(cfg.duration);
+    stop = 1;
+
     for (uint64_t i = 0; i < cfg.threads; i++) {
         thread *t = &threads[i];
         pthread_join(t->thread, NULL);
@@ -184,9 +165,15 @@ int main(int argc, char **argv) {
     long double req_per_s   = complete   / runtime_s;
     long double bytes_per_s = bytes      / runtime_s;
 
+    if (complete / cfg.connections > 0) {
+        int64_t interval = runtime_us / (complete / cfg.connections);
+        stats_correct(statistics.latency, interval);
+    }
+
     print_stats_header();
     print_stats("Latency", statistics.latency, format_time_us);
     print_stats("Req/Sec", statistics.requests, format_metric);
+    if (cfg.latency) print_stats_latency(statistics.latency);
 
     char *runtime_msg = format_time_us(runtime_us);
 
@@ -203,27 +190,39 @@ int main(int argc, char **argv) {
     printf("Requests/sec: %9.2Lf\n", req_per_s);
     printf("Transfer/sec: %10sB\n", format_binary(bytes_per_s));
 
+    if (script_has_done(L)) {
+        script_summary(L, runtime_us, complete, bytes);
+        script_errors(L, &errors);
+        script_done(L, statistics.latency, statistics.requests);
+    }
+
     return 0;
 }
 
 void *thread_main(void *arg) {
     thread *thread = arg;
 
-    aeEventLoop *loop = aeCreateEventLoop(10 + cfg.connections * 3);
+    char *request = NULL;
-    thread->cs   = zmalloc(thread->connections * sizeof(connection));
+    size_t length = 0;
-    thread->loop = loop;
-    tinymt64_init(&thread->rand, time_us());
 
+    if (!cfg.dynamic) {
+        script_request(thread->L, &request, &length);
+    }
+
+    thread->cs = zcalloc(thread->connections * sizeof(connection));
     connection *c = thread->cs;
 
     for (uint64_t i = 0; i < thread->connections; i++, c++) {
-        c->thread  = thread;
+        c->thread = thread;
-        c->latency = 0;
+        c->ssl     = cfg.ctx ? SSL_new(cfg.ctx) : NULL;
+        c->request = request;
+        c->length  = length;
+        c->delayed = cfg.delay;
         connect_socket(thread, c);
     }
 
-    aeCreateTimeEvent(loop, SAMPLE_INTERVAL_MS, sample_rate, thread, NULL);
+    aeEventLoop *loop = thread->loop;
-    aeCreateTimeEvent(loop, TIMEOUT_INTERVAL_MS, check_timeouts, thread, NULL);
+    aeCreateTimeEvent(loop, RECORD_INTERVAL_MS, record_rate, thread, NULL);
 
     thread->start = time_us();
     aeMain(loop);
@@ -235,136 +234,211 @@ void *thread_main(void *arg) {
 }
 
 static int connect_socket(thread *thread, connection *c) {
-    struct addrinfo addr = cfg.addr;
+    struct addrinfo *addr = thread->addr;
     struct aeEventLoop *loop = thread->loop;
     int fd, flags;
 
-    fd = socket(addr.ai_family, addr.ai_socktype, addr.ai_protocol);
+    fd = socket(addr->ai_family, addr->ai_socktype, addr->ai_protocol);
 
     flags = fcntl(fd, F_GETFL, 0);
     fcntl(fd, F_SETFL, flags | O_NONBLOCK);
 
-    if (connect(fd, addr.ai_addr, addr.ai_addrlen) == -1) {
+    if (connect(fd, addr->ai_addr, addr->ai_addrlen) == -1) {
-        if (errno != EINPROGRESS) {
+        if (errno != EINPROGRESS) goto error;
-            thread->errors.connect++;
-            goto error;
-        }
     }
 
     flags = 1;
     setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &flags, sizeof(flags));
 
-    if (aeCreateFileEvent(loop, fd, AE_WRITABLE, socket_writeable, c) != AE_OK) {
+    flags = AE_READABLE | AE_WRITABLE;
-        goto error;
+    if (aeCreateFileEvent(loop, fd, flags, socket_connected, c) == AE_OK) {
+        c->parser.data = c;
+        c->fd = fd;
+        return fd;
     }
 
-    http_parser_init(&c->parser, HTTP_RESPONSE);
-    c->parser.data = c;
-    c->fd = fd;
-
-    return fd;
-
   error:
+    thread->errors.connect++;
     close(fd);
     return -1;
 }
 
 static int reconnect_socket(thread *thread, connection *c) {
     aeDeleteFileEvent(thread->loop, c->fd, AE_WRITABLE | AE_READABLE);
+    sock.close(c);
     close(c->fd);
     return connect_socket(thread, c);
 }
 
-static int sample_rate(aeEventLoop *loop, long long id, void *data) {
+static int record_rate(aeEventLoop *loop, long long id, void *data) {
     thread *thread = data;
 
-    uint64_t n = rand64(&thread->rand, thread->connections);
+    if (thread->requests > 0) {
-    uint64_t elapsed_ms = (time_us() - thread->start) / 1000;
+        uint64_t elapsed_ms = (time_us() - thread->start) / 1000;
-    connection *c = thread->cs + n;
+        uint64_t requests = (thread->requests / (double) elapsed_ms) * 1000;
-    uint64_t requests = (thread->complete / elapsed_ms) * 1000;
 
-    pthread_mutex_lock(&statistics.mutex);
+        stats_record(statistics.requests, requests);
-    stats_record(statistics.latency,  c->latency);
-    stats_record(statistics.requests, requests);
-    pthread_mutex_unlock(&statistics.mutex);
 
-    return SAMPLE_INTERVAL_MS + rand64(&thread->rand, SAMPLE_INTERVAL_MS);
+        thread->requests = 0;
+        thread->start    = time_us();
+    }
+
+    if (stop) aeStop(loop);
+
+    return RECORD_INTERVAL_MS;
 }
 
-static int request_complete(http_parser *parser) {
+static int delay_request(aeEventLoop *loop, long long id, void *data) {
+    connection *c = data;
+    c->delayed = false;
+    aeCreateFileEvent(loop, c->fd, AE_WRITABLE, socket_writeable, c);
+    return AE_NOMORE;
+}
+
+static int header_field(http_parser *parser, const char *at, size_t len) {
+    connection *c = parser->data;
+    if (c->state == VALUE) {
+        *c->headers.cursor++ = '\0';
+        c->state = FIELD;
+    }
+    buffer_append(&c->headers, at, len);
+    return 0;
+}
+
+static int header_value(http_parser *parser, const char *at, size_t len) {
+    connection *c = parser->data;
+    if (c->state == FIELD) {
+        *c->headers.cursor++ = '\0';
+        c->state = VALUE;
+    }
+    buffer_append(&c->headers, at, len);
+    return 0;
+}
+
+static int response_body(http_parser *parser, const char *at, size_t len) {
+    connection *c = parser->data;
+    buffer_append(&c->body, at, len);
+    return 0;
+}
+
+static int response_complete(http_parser *parser) {
     connection *c = parser->data;
     thread *thread = c->thread;
     uint64_t now = time_us();
+    int status = parser->status_code;
 
     thread->complete++;
-    c->latency = now - c->start;
+    thread->requests++;
 
-    if (parser->status_code > 399) {
+    if (status > 399) {
         thread->errors.status++;
     }
 
-    if (now >= thread->stop_at) {
+    if (c->headers.buffer) {
-        aeStop(thread->loop);
+        *c->headers.cursor++ = '\0';
+        script_response(thread->L, status, &c->headers, &c->body);
+        c->state = FIELD;
+    }
+
+    if (--c->pending == 0) {
+        if (!stats_record(statistics.latency, now - c->start)) {
+            thread->errors.timeout++;
+        }
+        c->delayed = cfg.delay;
+        aeCreateFileEvent(thread->loop, c->fd, AE_WRITABLE, socket_writeable, c);
+    }
+
+    if (!http_should_keep_alive(parser)) {
+        reconnect_socket(thread, c);
         goto done;
     }
 
-    if (!http_should_keep_alive(parser)) goto reconnect;
-
     http_parser_init(parser, HTTP_RESPONSE);
-    aeDeleteFileEvent(thread->loop, c->fd, AE_READABLE);
-    aeCreateFileEvent(thread->loop, c->fd, AE_WRITABLE, socket_writeable, c);
-
-    goto done;
-
-  reconnect:
-    reconnect_socket(thread, c);
 
   done:
     return 0;
 }
 
-static int check_timeouts(aeEventLoop *loop, long long id, void *data) {
+static void socket_connected(aeEventLoop *loop, int fd, void *data, int mask) {
-    thread *thread = data;
-    connection *c  = thread->cs;
-    uint64_t now   = time_us();
-
-    uint64_t maxAge = now - (cfg.timeout * 1000);
-
-    for (uint64_t i = 0; i < thread->connections; i++, c++) {
-        if (maxAge > c->start) {
-            thread->errors.timeout++;
-        }
-    }
-
-    if (stop || now >= thread->stop_at) {
-        aeStop(loop);
-    }
-
-    return TIMEOUT_INTERVAL_MS;
-}
-
-static void socket_writeable(aeEventLoop *loop, int fd, void *data, int mask) {
     connection *c = data;
 
-    if (write(fd, request.buf, request.size) < request.size) goto error;
+    switch (sock.connect(c, cfg.host)) {
-    c->start = time_us();
+        case OK:    break;
-    aeDeleteFileEvent(loop, fd, AE_WRITABLE);
+        case ERROR: goto error;
-    aeCreateFileEvent(loop, fd, AE_READABLE, socket_readable, c);
+        case RETRY: return;
+    }
+
+    http_parser_init(&c->parser, HTTP_RESPONSE);
+    c->written = 0;
+
+    aeCreateFileEvent(c->thread->loop, fd, AE_READABLE, socket_readable, c);
+    aeCreateFileEvent(c->thread->loop, fd, AE_WRITABLE, socket_writeable, c);
 
     return;
 
   error:
-    c->thread->errors.write++;
+    c->thread->errors.connect++;
     reconnect_socket(c->thread, c);
 }
 
+static void socket_writeable(aeEventLoop *loop, int fd, void *data, int mask) {
+    connection *c = data;
+    thread *thread = c->thread;
+
+    if (c->delayed) {
+        uint64_t delay = script_delay(thread->L);
+        aeDeleteFileEvent(loop, fd, AE_WRITABLE);
+        aeCreateTimeEvent(loop, delay, delay_request, c, NULL);
+        return;
+    }
+
+    if (!c->written) {
+        if (cfg.dynamic) {
+            script_request(thread->L, &c->request, &c->length);
+        }
+        c->start   = time_us();
+        c->pending = cfg.pipeline;
+    }
+
+    char  *buf = c->request + c->written;
+    size_t len = c->length  - c->written;
+    size_t n;
+
+    switch (sock.write(c, buf, len, &n)) {
+        case OK:    break;
+        case ERROR: goto error;
+        case RETRY: return;
+    }
+
+    c->written += n;
+    if (c->written == c->length) {
+        c->written = 0;
+        aeDeleteFileEvent(loop, fd, AE_WRITABLE);
+    }
+
+    return;
+
+  error:
+    thread->errors.write++;
+    reconnect_socket(thread, c);
+}
+
 static void socket_readable(aeEventLoop *loop, int fd, void *data, int mask) {
     connection *c = data;
-    ssize_t n;
+    size_t n;
 
-    if ((n = read(fd, c->buf, sizeof(c->buf))) <= 0) goto error;
+    do {
-    if (http_parser_execute(&c->parser, &parser_settings, c->buf, n) != n) goto error;
+        switch (sock.read(c, &n)) {
-    c->thread->bytes += n;
+            case OK:    break;
+            case ERROR: goto error;
+            case RETRY: return;
+        }
+
+        if (http_parser_execute(&c->parser, &parser_settings, c->buf, n) != n) goto error;
+        if (n == 0 && !http_body_is_final(&c->parser)) goto error;
+
+        c->thread->bytes += n;
+    } while (n == RECVBUF && sock.readable(c) > 0);
 
     return;
 
@@ -379,21 +453,12 @@ static uint64_t time_us() {
     return (t.tv_sec * 1000000) + t.tv_usec;
 }
 
-static uint64_t rand64(tinymt64_t *state, uint64_t n) {
+static char *copy_url_part(char *url, struct http_parser_url *parts, enum http_parser_url_fields field) {
-    uint64_t x, max = ~UINT64_C(0);
-    max -= max % n;
-    do {
-        x = tinymt64_generate_uint64(state);
-    } while (x >= max);
-    return x % n;
-}
-
-static char *extract_url_part(char *url, struct http_parser_url *parser_url, enum http_parser_url_fields field) {
     char *part = NULL;
 
-    if (parser_url->field_set & (1 << field)) {
+    if (parts->field_set & (1 << field)) {
-        uint16_t off = parser_url->field_data[field].off;
+        uint16_t off = parts->field_data[field].off;
-        uint16_t len = parser_url->field_data[field].len;
+        uint16_t len = parts->field_data[field].len;
         part = zcalloc(len + 1 * sizeof(char));
         memcpy(part, &url[off], len);
     }
@@ -401,41 +466,22 @@ static char *extract_url_part(char *url, struct http_parser_url *parser_url, enu
     return part;
 }
 
-static char *format_request(char *host, char *port, char *path, char **headers) {
-    char *req  = NULL;
-    char *head = NULL;
-
-    for (char **h = headers; *h != NULL; h++) {
-        aprintf(&head, "%s\r\n", *h);
-        if (!strncasecmp(*h, "Host:", 5)) {
-            host = NULL;
-            port = NULL;
-        }
-    }
-
-    aprintf(&req, "GET %s HTTP/1.1\r\n", path);
-    if (host) aprintf(&req, "Host: %s", host);
-    if (port) aprintf(&req, ":%s", port);
-    if (host) aprintf(&req, "\r\n");
-    aprintf(&req, "%s\r\n", head ? head : "");
-
-    free(head);
-    return req;
-}
-
 static struct option longopts[] = {
     { "connections", required_argument, NULL, 'c' },
     { "duration",    required_argument, NULL, 'd' },
     { "threads",     required_argument, NULL, 't' },
+    { "script",      required_argument, NULL, 's' },
     { "header",      required_argument, NULL, 'H' },
+    { "latency",     no_argument,       NULL, 'L' },
     { "timeout",     required_argument, NULL, 'T' },
     { "help",        no_argument,       NULL, 'h' },
     { "version",     no_argument,       NULL, 'v' },
     { NULL,          0,                 NULL,  0  }
 };
 
-static int parse_args(struct config *cfg, char **url, char **headers, int argc, char **argv) {
+static int parse_args(struct config *cfg, char **url, struct http_parser_url *parts, char **headers, int argc, char **argv) {
-    char c, **header = headers;
+    char **header = headers;
+    int c;
 
     memset(cfg, 0, sizeof(struct config));
     cfg->threads     = 2;
@@ -443,7 +489,7 @@ static int parse_args(struct config *cfg, char **url, char **headers, int argc,
     cfg->duration    = 10;
     cfg->timeout     = SOCKET_TIMEOUT_MS;
 
-    while ((c = getopt_long(argc, argv, "t:c:d:H:T:rv?", longopts, NULL)) != -1) {
+    while ((c = getopt_long(argc, argv, "t:c:d:s:H:T:Lrv?", longopts, NULL)) != -1) {
         switch (c) {
             case 't':
                 if (scan_metric(optarg, &cfg->threads)) return -1;
@@ -454,9 +500,15 @@ static int parse_args(struct config *cfg, char **url, char **headers, int argc,
             case 'd':
                 if (scan_time(optarg, &cfg->duration)) return -1;
                 break;
+            case 's':
+                cfg->script = optarg;
+                break;
             case 'H':
                 *header++ = optarg;
                 break;
+            case 'L':
+                cfg->latency = true;
+                break;
             case 'T':
                 if (scan_time(optarg, &cfg->timeout)) return -1;
                 cfg->timeout *= 1000;
@@ -465,8 +517,6 @@ static int parse_args(struct config *cfg, char **url, char **headers, int argc,
                 printf("wrk %s [%s] ", VERSION, aeGetApiName());
                 printf("Copyright (C) 2012 Will Glozer\n");
                 break;
-            case 'r':
-                fprintf(stderr, "wrk 2.0.0+ uses -d instead of -r\n");
             case 'h':
             case '?':
             case ':':
@@ -477,6 +527,11 @@ static int parse_args(struct config *cfg, char **url, char **headers, int argc,
 
     if (optind == argc || !cfg->threads || !cfg->duration) return -1;
 
+    if (!script_parse_url(argv[optind], parts)) {
+        fprintf(stderr, "invalid URL: %s\n", argv[optind]);
+        return -1;
+    }
+
     if (!cfg->connections || cfg->connections < cfg->threads) {
         fprintf(stderr, "number of connections must be >= threads\n");
         return -1;
@@ -506,8 +561,8 @@ static void print_units(long double n, char *(*fmt)(long double), int width) {
 }
 
 static void print_stats(char *name, stats *stats, char *(*fmt)(long double)) {
+    uint64_t max = stats->max;
     long double mean  = stats_mean(stats);
-    long double max   = stats_max(stats);
     long double stdev = stats_stdev(stats, mean);
 
     printf("    %-10s", name);
@@ -516,3 +571,15 @@ static void print_stats(char *name, stats *stats, char *(*fmt)(long double)) {
     print_units(max,   fmt, 9);
     printf("%8.2Lf%%\n", stats_within_stdev(stats, mean, stdev, 1));
 }
+
+static void print_stats_latency(stats *stats) {
+    long double percentiles[] = { 50.0, 75.0, 90.0, 99.0 };
+    printf("  Latency Distribution\n");
+    for (size_t i = 0; i < sizeof(percentiles) / sizeof(long double); i++) {
+        long double p = percentiles[i];
+        uint64_t n = stats_percentile(stats, p);
+        printf("%7.0Lf%%", p);
+        print_units(n, format_time_us, 10);
+        printf("\n");
+    }
+}

src/wrk.h

@@ -5,71 +5,62 @@
 #include <pthread.h>
 #include <inttypes.h>
 #include <sys/types.h>
+#include <netdb.h>
+#include <sys/socket.h>
+
+#include <openssl/ssl.h>
+#include <openssl/err.h>
+#include <lua.h>
 
 #include "stats.h"
 #include "ae.h"
 #include "http_parser.h"
-#include "tinymt64.h"
 
-#define VERSION  "2.0.0"
 #define RECVBUF  8192
-#define SAMPLES  100000
 
+#define MAX_THREAD_RATE_S   10000000
 #define SOCKET_TIMEOUT_MS   2000
-#define SAMPLE_INTERVAL_MS  100
+#define RECORD_INTERVAL_MS  100
-#define TIMEOUT_INTERVAL_MS 2000
 
-typedef struct {
+extern const char *VERSION;
-    uint32_t connect;
-    uint32_t read;
-    uint32_t write;
-    uint32_t status;
-    uint32_t timeout;
-} errors;
 
 typedef struct {
     pthread_t thread;
     aeEventLoop *loop;
+    struct addrinfo *addr;
     uint64_t connections;
-    uint64_t stop_at;
     uint64_t complete;
+    uint64_t requests;
     uint64_t bytes;
     uint64_t start;
-    tinymt64_t rand;
+    lua_State *L;
     errors errors;
     struct connection *cs;
 } thread;
 
+typedef struct {
+    char  *buffer;
+    size_t length;
+    char  *cursor;
+} buffer;
+
 typedef struct connection {
     thread *thread;
     http_parser parser;
+    enum {
+        FIELD, VALUE
+    } state;
     int fd;
+    SSL *ssl;
+    bool delayed;
     uint64_t start;
-    uint64_t latency;
+    char *request;
+    size_t length;
+    size_t written;
+    uint64_t pending;
+    buffer headers;
+    buffer body;
     char buf[RECVBUF];
 } connection;
 
-struct config;
-
-static void *thread_main(void *);
-static int connect_socket(thread *, connection *);
-static int reconnect_socket(thread *, connection *);
-
-static int sample_rate(aeEventLoop *, long long, void *);
-static int check_timeouts(aeEventLoop *, long long, void *);
-
-static void socket_writeable(aeEventLoop *, int, void *, int);
-static void socket_readable(aeEventLoop *, int, void *, int);
-static int request_complete(http_parser *);
-
-static uint64_t time_us();
-static uint64_t rand64(tinymt64_t *, uint64_t);
-
-static char *extract_url_part(char *, struct http_parser_url *, enum http_parser_url_fields);
-static char *format_request(char *, char *, char *, char **);
-
-static int parse_args(struct config *, char **, char **, int, char **);
-static void print_stats_header();
-static void print_stats(char *, stats *, char *(*)(long double));
-
 #endif /* WRK_H */

src/wrk.lua

@@ -0,0 +1,74 @@
+local wrk = {
+   scheme  = "http",
+   host    = "localhost",
+   port    = nil,
+   method  = "GET",
+   path    = "/",
+   headers = {},
+   body    = nil,
+   thread  = nil,
+}
+
+function wrk.resolve(host, service)
+   local addrs = wrk.lookup(host, service)
+   for i = #addrs, 1, -1 do
+      if not wrk.connect(addrs[i]) then
+         table.remove(addrs, i)
+      end
+   end
+   wrk.addrs = addrs
+end
+
+function wrk.setup(thread)
+   thread.addr = wrk.addrs[1]
+   if type(setup) == "function" then
+      setup(thread)
+   end
+end
+
+function wrk.init(args)
+   if not wrk.headers["Host"] then
+      local host = wrk.host
+      local port = wrk.port
+
+      host = host:find(":") and ("[" .. host .. "]")  or host
+      host = port           and (host .. ":" .. port) or host
+
+      wrk.headers["Host"] = host
+   end
+
+   if type(init) == "function" then
+      init(args)
+   end
+
+   local req = wrk.format()
+   wrk.request = function()
+      return req
+   end
+end
+
+function wrk.format(method, path, headers, body)
+   local method  = method  or wrk.method
+   local path    = path    or wrk.path
+   local headers = headers or wrk.headers
+   local body    = body    or wrk.body
+   local s       = {}
+
+   if not headers["Host"] then
+      headers["Host"] = wrk.headers["Host"]
+   end
+
+   headers["Content-Length"] = body and string.len(body)
+
+   s[1] = string.format("%s %s HTTP/1.1", method, path)
+   for name, value in pairs(headers) do
+      s[#s+1] = string.format("%s: %s", name, value)
+   end
+
+   s[#s+1] = ""
+   s[#s+1] = body or ""
+
+   return table.concat(s, "\r\n")
+end
+
+return wrk

src/zmalloc.c

@@ -30,10 +30,20 @@
 
 #include <stdio.h>
 #include <stdlib.h>
+
+/* This function provide us access to the original libc free(). This is useful
+ * for instance to free results obtained by backtrace_symbols(). We need
+ * to define this function before including zmalloc.h that may shadow the
+ * free implementation if we use jemalloc or another non standard allocator. */
+void zlibc_free(void *ptr) {
+    free(ptr);
+}
+
 #include <string.h>
 #include <pthread.h>
 #include "config.h"
 #include "zmalloc.h"
+#include "atomicvar.h"
 
 #ifdef HAVE_MALLOC_SIZE
 #define PREFIX_SIZE (0)
@@ -56,67 +66,76 @@
 #define calloc(count,size) je_calloc(count,size)
 #define realloc(ptr,size) je_realloc(ptr,size)
 #define free(ptr) je_free(ptr)
+#define mallocx(size,flags) je_mallocx(size,flags)
+#define dallocx(ptr,flags) je_dallocx(ptr,flags)
 #endif
 
-#define update_zmalloc_stat_alloc(__n,__size) do { \
+#define update_zmalloc_stat_alloc(__n) do { \
     size_t _n = (__n); \
     if (_n&(sizeof(long)-1)) _n += sizeof(long)-(_n&(sizeof(long)-1)); \
-    if (zmalloc_thread_safe) { \
+    atomicIncr(used_memory,__n); \
-        pthread_mutex_lock(&used_memory_mutex);  \
-        used_memory += _n; \
-        pthread_mutex_unlock(&used_memory_mutex); \
-    } else { \
-        used_memory += _n; \
-    } \
 } while(0)
 
 #define update_zmalloc_stat_free(__n) do { \
     size_t _n = (__n); \
     if (_n&(sizeof(long)-1)) _n += sizeof(long)-(_n&(sizeof(long)-1)); \
-    if (zmalloc_thread_safe) { \
+    atomicDecr(used_memory,__n); \
-        pthread_mutex_lock(&used_memory_mutex);  \
-        used_memory -= _n; \
-        pthread_mutex_unlock(&used_memory_mutex); \
-    } else { \
-        used_memory -= _n; \
-    } \
 } while(0)
 
 static size_t used_memory = 0;
-static int zmalloc_thread_safe = 0;
 pthread_mutex_t used_memory_mutex = PTHREAD_MUTEX_INITIALIZER;
 
-static void zmalloc_oom(size_t size) {
+static void zmalloc_default_oom(size_t size) {
     fprintf(stderr, "zmalloc: Out of memory trying to allocate %zu bytes\n",
         size);
     fflush(stderr);
     abort();
 }
 
+static void (*zmalloc_oom_handler)(size_t) = zmalloc_default_oom;
+
 void *zmalloc(size_t size) {
     void *ptr = malloc(size+PREFIX_SIZE);
 
-    if (!ptr) zmalloc_oom(size);
+    if (!ptr) zmalloc_oom_handler(size);
 #ifdef HAVE_MALLOC_SIZE
-    update_zmalloc_stat_alloc(zmalloc_size(ptr),size);
+    update_zmalloc_stat_alloc(zmalloc_size(ptr));
     return ptr;
 #else
     *((size_t*)ptr) = size;
-    update_zmalloc_stat_alloc(size+PREFIX_SIZE,size);
+    update_zmalloc_stat_alloc(size+PREFIX_SIZE);
     return (char*)ptr+PREFIX_SIZE;
 #endif
 }
 
+/* Allocation and free functions that bypass the thread cache
+ * and go straight to the allocator arena bins.
+ * Currently implemented only for jemalloc. Used for online defragmentation. */
+#ifdef HAVE_DEFRAG
+void *zmalloc_no_tcache(size_t size) {
+    void *ptr = mallocx(size+PREFIX_SIZE, MALLOCX_TCACHE_NONE);
+    if (!ptr) zmalloc_oom_handler(size);
+    update_zmalloc_stat_alloc(zmalloc_size(ptr));
+    return ptr;
+}
+
+void zfree_no_tcache(void *ptr) {
+    if (ptr == NULL) return;
+    update_zmalloc_stat_free(zmalloc_size(ptr));
+    dallocx(ptr, MALLOCX_TCACHE_NONE);
+}
+#endif
+
 void *zcalloc(size_t size) {
     void *ptr = calloc(1, size+PREFIX_SIZE);
 
-    if (!ptr) zmalloc_oom(size);
+    if (!ptr) zmalloc_oom_handler(size);
 #ifdef HAVE_MALLOC_SIZE
-    update_zmalloc_stat_alloc(zmalloc_size(ptr),size);
+    update_zmalloc_stat_alloc(zmalloc_size(ptr));
     return ptr;
 #else
     *((size_t*)ptr) = size;
-    update_zmalloc_stat_alloc(size+PREFIX_SIZE,size);
+    update_zmalloc_stat_alloc(size+PREFIX_SIZE);
     return (char*)ptr+PREFIX_SIZE;
 #endif
 }
@@ -132,26 +151,26 @@ void *zrealloc(void *ptr, size_t size) {
 #ifdef HAVE_MALLOC_SIZE
     oldsize = zmalloc_size(ptr);
     newptr = realloc(ptr,size);
-    if (!newptr) zmalloc_oom(size);
+    if (!newptr) zmalloc_oom_handler(size);
 
     update_zmalloc_stat_free(oldsize);
-    update_zmalloc_stat_alloc(zmalloc_size(newptr),size);
+    update_zmalloc_stat_alloc(zmalloc_size(newptr));
     return newptr;
 #else
     realptr = (char*)ptr-PREFIX_SIZE;
     oldsize = *((size_t*)realptr);
     newptr = realloc(realptr,size+PREFIX_SIZE);
-    if (!newptr) zmalloc_oom(size);
+    if (!newptr) zmalloc_oom_handler(size);
 
     *((size_t*)newptr) = size;
     update_zmalloc_stat_free(oldsize);
-    update_zmalloc_stat_alloc(size,size);
+    update_zmalloc_stat_alloc(size);
     return (char*)newptr+PREFIX_SIZE;
 #endif
 }
 
 /* Provide zmalloc_size() for systems where this function is not provided by
- * malloc itself, given that in that case we store an header with this
+ * malloc itself, given that in that case we store a header with this
  * information as the first bytes of every allocation. */
 #ifndef HAVE_MALLOC_SIZE
 size_t zmalloc_size(void *ptr) {
@@ -192,15 +211,12 @@ char *zstrdup(const char *s) {
 
 size_t zmalloc_used_memory(void) {
     size_t um;
-
+    atomicGet(used_memory,um);
-    if (zmalloc_thread_safe) pthread_mutex_lock(&used_memory_mutex);
-    um = used_memory;
-    if (zmalloc_thread_safe) pthread_mutex_unlock(&used_memory_mutex);
     return um;
 }
 
-void zmalloc_enable_thread_safeness(void) {
+void zmalloc_set_oom_handler(void (*oom_handler)(size_t)) {
-    zmalloc_thread_safe = 1;
+    zmalloc_oom_handler = oom_handler;
 }
 
 /* Get the RSS information in an OS-specific way.
@@ -211,9 +227,9 @@ void zmalloc_enable_thread_safeness(void) {
  *
  * For this kind of "fast RSS reporting" usages use instead the
  * function RedisEstimateRSS() that is a much faster (and less precise)
- * version of the funciton. */
+ * version of the function. */
 
-#if defined(HAVE_PROCFS)
+#if defined(HAVE_PROC_STAT)
 #include <unistd.h>
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -282,6 +298,114 @@ size_t zmalloc_get_rss(void) {
 #endif
 
 /* Fragmentation = RSS / allocated-bytes */
-float zmalloc_get_fragmentation_ratio(void) {
+float zmalloc_get_fragmentation_ratio(size_t rss) {
-    return (float)zmalloc_get_rss()/zmalloc_used_memory();
+    return (float)rss/zmalloc_used_memory();
 }
+
+/* Get the sum of the specified field (converted form kb to bytes) in
+ * /proc/self/smaps. The field must be specified with trailing ":" as it
+ * apperas in the smaps output.
+ *
+ * If a pid is specified, the information is extracted for such a pid,
+ * otherwise if pid is -1 the information is reported is about the
+ * current process.
+ *
+ * Example: zmalloc_get_smap_bytes_by_field("Rss:",-1);
+ */
+#if defined(HAVE_PROC_SMAPS)
+size_t zmalloc_get_smap_bytes_by_field(char *field, long pid) {
+    char line[1024];
+    size_t bytes = 0;
+    int flen = strlen(field);
+    FILE *fp;
+
+    if (pid == -1) {
+        fp = fopen("/proc/self/smaps","r");
+    } else {
+        char filename[128];
+        snprintf(filename,sizeof(filename),"/proc/%ld/smaps",pid);
+        fp = fopen(filename,"r");
+    }
+
+    if (!fp) return 0;
+    while(fgets(line,sizeof(line),fp) != NULL) {
+        if (strncmp(line,field,flen) == 0) {
+            char *p = strchr(line,'k');
+            if (p) {
+                *p = '\0';
+                bytes += strtol(line+flen,NULL,10) * 1024;
+            }
+        }
+    }
+    fclose(fp);
+    return bytes;
+}
+#else
+size_t zmalloc_get_smap_bytes_by_field(char *field, long pid) {
+    ((void) field);
+    ((void) pid);
+    return 0;
+}
+#endif
+
+size_t zmalloc_get_private_dirty(long pid) {
+    return zmalloc_get_smap_bytes_by_field("Private_Dirty:",pid);
+}
+
+/* Returns the size of physical memory (RAM) in bytes.
+ * It looks ugly, but this is the cleanest way to achive cross platform results.
+ * Cleaned up from:
+ *
+ * http://nadeausoftware.com/articles/2012/09/c_c_tip_how_get_physical_memory_size_system
+ *
+ * Note that this function:
+ * 1) Was released under the following CC attribution license:
+ *    http://creativecommons.org/licenses/by/3.0/deed.en_US.
+ * 2) Was originally implemented by David Robert Nadeau.
+ * 3) Was modified for Redis by Matt Stancliff.
+ * 4) This note exists in order to comply with the original license.
+ */
+size_t zmalloc_get_memory_size(void) {
+#if defined(__unix__) || defined(__unix) || defined(unix) || \
+    (defined(__APPLE__) && defined(__MACH__))
+#if defined(CTL_HW) && (defined(HW_MEMSIZE) || defined(HW_PHYSMEM64))
+    int mib[2];
+    mib[0] = CTL_HW;
+#if defined(HW_MEMSIZE)
+    mib[1] = HW_MEMSIZE;            /* OSX. --------------------- */
+#elif defined(HW_PHYSMEM64)
+    mib[1] = HW_PHYSMEM64;          /* NetBSD, OpenBSD. --------- */
+#endif
+    int64_t size = 0;               /* 64-bit */
+    size_t len = sizeof(size);
+    if (sysctl( mib, 2, &size, &len, NULL, 0) == 0)
+        return (size_t)size;
+    return 0L;          /* Failed? */
+
+#elif defined(_SC_PHYS_PAGES) && defined(_SC_PAGESIZE)
+    /* FreeBSD, Linux, OpenBSD, and Solaris. -------------------- */
+    return (size_t)sysconf(_SC_PHYS_PAGES) * (size_t)sysconf(_SC_PAGESIZE);
+
+#elif defined(CTL_HW) && (defined(HW_PHYSMEM) || defined(HW_REALMEM))
+    /* DragonFly BSD, FreeBSD, NetBSD, OpenBSD, and OSX. -------- */
+    int mib[2];
+    mib[0] = CTL_HW;
+#if defined(HW_REALMEM)
+    mib[1] = HW_REALMEM;        /* FreeBSD. ----------------- */
+#elif defined(HW_PYSMEM)
+    mib[1] = HW_PHYSMEM;        /* Others. ------------------ */
+#endif
+    unsigned int size = 0;      /* 32-bit */
+    size_t len = sizeof(size);
+    if (sysctl(mib, 2, &size, &len, NULL, 0) == 0)
+        return (size_t)size;
+    return 0L;          /* Failed? */
+#else
+    return 0L;          /* Unknown method to get the data. */
+#endif
+#else
+    return 0L;          /* Unknown OS. */
+#endif
+}
+
+

src/zmalloc.h

@@ -38,7 +38,7 @@
 #if defined(USE_TCMALLOC)
 #define ZMALLOC_LIB ("tcmalloc-" __xstr(TC_VERSION_MAJOR) "." __xstr(TC_VERSION_MINOR))
 #include <google/tcmalloc.h>
-#if TC_VERSION_MAJOR >= 1 && TC_VERSION_MINOR >= 6
+#if (TC_VERSION_MAJOR == 1 && TC_VERSION_MINOR >= 6) || (TC_VERSION_MAJOR > 1)
 #define HAVE_MALLOC_SIZE 1
 #define zmalloc_size(p) tc_malloc_size(p)
 #else
@@ -47,11 +47,10 @@
 
 #elif defined(USE_JEMALLOC)
 #define ZMALLOC_LIB ("jemalloc-" __xstr(JEMALLOC_VERSION_MAJOR) "." __xstr(JEMALLOC_VERSION_MINOR) "." __xstr(JEMALLOC_VERSION_BUGFIX))
-#define JEMALLOC_MANGLE
 #include <jemalloc/jemalloc.h>
-#if JEMALLOC_VERSION_MAJOR >= 2 && JEMALLOC_VERSION_MINOR >= 1
+#if (JEMALLOC_VERSION_MAJOR == 2 && JEMALLOC_VERSION_MINOR >= 1) || (JEMALLOC_VERSION_MAJOR > 2)
 #define HAVE_MALLOC_SIZE 1
-#define zmalloc_size(p) JEMALLOC_P(malloc_usable_size)(p)
+#define zmalloc_size(p) je_malloc_usable_size(p)
 #else
 #error "Newer version of jemalloc required"
 #endif
@@ -66,15 +65,31 @@
 #define ZMALLOC_LIB "libc"
 #endif
 
+/* We can enable the Redis defrag capabilities only if we are using Jemalloc
+ * and the version used is our special version modified for Redis having
+ * the ability to return per-allocation fragmentation hints. */
+#if defined(USE_JEMALLOC) && defined(JEMALLOC_FRAG_HINT)
+#define HAVE_DEFRAG
+#endif
+
 void *zmalloc(size_t size);
 void *zcalloc(size_t size);
 void *zrealloc(void *ptr, size_t size);
 void zfree(void *ptr);
 char *zstrdup(const char *s);
 size_t zmalloc_used_memory(void);
-void zmalloc_enable_thread_safeness(void);
+void zmalloc_set_oom_handler(void (*oom_handler)(size_t));
-float zmalloc_get_fragmentation_ratio(void);
+float zmalloc_get_fragmentation_ratio(size_t rss);
 size_t zmalloc_get_rss(void);
+size_t zmalloc_get_private_dirty(long pid);
+size_t zmalloc_get_smap_bytes_by_field(char *field, long pid);
+size_t zmalloc_get_memory_size(void);
+void zlibc_free(void *ptr);
+
+#ifdef HAVE_DEFRAG
+void zfree_no_tcache(void *ptr);
+void *zmalloc_no_tcache(size_t size);
+#endif
 
 #ifndef HAVE_MALLOC_SIZE
 size_t zmalloc_size(void *ptr);