Http0.7.0 -> master (#122)

modified: WebSockets.jl  move HTTP imports to HTTP.jl
modified: HTTP.jl  Import 25 functions and types from 0.7.0 locations
              Use imports without qualifications.
			  Duck-type _openstream 
              'http' -> 'stream' or 'r', depending on what it can be.
			  Use 'getrawstream', since stream now can be a Transaction.
			  Don't chek for excess bytes, remove rest of issue #114 check
			  Update inline listen example.
			  Reintroduce rethrowing errors from upgrade
			  'Where called from' for functions inline docs.
			  Add alternative method for is_upgrade for a stream.
              Added own version of ServerOptions due to missing union type in HTTP. The union type prevents malfunction with MbedTLS types.
			  Removed dead 'while' loop in serve(..)
			  New union type for server reference. 
			  Attempts to control garbage collection when releasing the safe reference. There are still warnings when a server is closed.
			  Anonymous coroutine in 'listen' -> named inner function '_servercoroutine'. 
			  Added own default throttling function 'checkratelimit!', avoid terminal clutter.
			  Remove type PatchedIO (issue #114).
modified: REQUIRE  -> HTTP 0.7.0 0.7.99
modified: test/listen_test.jl Restructure imports
modified: test/listen_test.jl Restructure imports
new: test/client_server_functions.jl
new: test/client_serverWS_test.jl	Extensive message tests	  
new: test/client_listen_test.jl	Same as serverWS, listen syntax
modified: test/error_test.jl Restructure imports
modified: test/frametest.jl Restructure imports
new: test/handshaketest_functions.jl
modified: test/handshaketest.jl
new: test/logformat.jl	Suppress HTTP.Server terminal output
					Add time and location logs
modified: test/REQUIRE Don't require HTTP as a separate package.
modified: test/runtests.jl  Blue test headers
new: test/throttling_test.jl
modified: .travis.yml	Don't test nightly
modified: appeyor.yml    Allow test failures on nightly
modified: chat_explore.jl Restructure imports
modified: chat_explore.jl Restructure imports, remove HTTP.Server messages
modified: client_repl_input.jl Restructure imports
modified: client_repl_input.jl Restructure imports
modified: minimal_server.jl  Restructure imports
modified: minimal_listen_do_syntax.jl  New syntax, imports
modified: minimal_server.jl  Restructure imports

.travis.yml

@@ -5,7 +5,6 @@ os:
 julia:
     - 0.7
     - 1.0
-    - nightly
 sudo: false
 notifications:
     email: false

README.md

@@ -2,83 +2,92 @@
 
 *Release version*:
 
-[![WebSockets](http://pkg.julialang.org/badges/WebSockets_0.6.svg)](http://pkg.julialang.org/?pkg=WebSockets&ver=0.6) [![Build Status](https://travis-ci.org/JuliaWeb/WebSockets.jl.svg)](https://travis-ci.org/JuliaWeb/WebSockets.jl)
-[![Coverage Status](https://img.shields.io/coveralls/JuliaWeb/WebSockets.jl.svg)](https://coveralls.io/r/JuliaWeb/WebSockets.jl)
+[![WebSockets](http://pkg.julialang.org/badges/WebSockets_0.6.svg)](http://pkg.julialang.org/?pkg=WebSockets&ver=0.6) [![Build Status](https://travis-ci.org/JuliaWeb/WebSockets.jl.svg)](https://travis-ci.org/JuliaWeb/WebSockets.jl)<!---
+Enable coverage when https://github.com/JuliaCI/Coverage.jl/issues/187 is resolved.
+[![Coverage Status](https://img.shields.io/coveralls/JuliaWeb/WebSockets.jl.svg)] (https://coveralls.io/r/JuliaWeb/WebSockets.jl)a --->
 
+Test coverage 96%
 
 *Development version*:
 
 [![WebSockets](http://pkg.julialang.org/badges/WebSockets_0.6.svg?branch?master)](http://pkg.julialang.org/?pkg=WebSockets&ver=0.6)
 [![Build Status](https://travis-ci.org/JuliaWeb/WebSockets.jl.svg?branch=master)](https://travis-ci.org/JuliaWeb/WebSockets.jl)
-[![Coverage Status](https://img.shields.io/coveralls/JuliaWeb/WebSockets.jl.svg?branch=master)](https://coveralls.io/r/JuliaWeb/WebSockets.jl?branch=master)
-[![Appveyor](https://ci.appveyor.com/api/projects/status/github/JuliaWeb/WebSockets.jl?svg=true&branch=master)](https://ci.appveyor.com/project/JuliaWeb/WebSockets-jl)
+<!---[![Coverage Status](https://img.shields.io/coveralls/JuliaWeb/WebSockets.jl.svg?branch=master)](https://coveralls.io/r/JuliaWeb/WebSockets.jl?branch=master)
+[![Appveyor](https://ci.appveyor.com/api/projects/status/github/JuliaWeb/WebSockets.jl?svg=true&branch=master)](https://ci.appveyor.com/project/JuliaWeb/WebSockets-jl)--->
 
+Test coverage 96%
 
 
 Server and client side [Websockets](https://tools.ietf.org/html/rfc6455) protocol in Julia. WebSockets is a small overhead message protocol layered over [TCP](https://tools.ietf.org/html/rfc793). It uses HTTP(S) for establishing the connections.
 
 ## Getting started
-On Julia pre 0.7, see an earlier version of this repository.
+Copy this into Julia:
 
 ```julia
-(v0.7) pkg>add WebSockets
+(v1.0) pkg> add WebSockets
 julia> using WebSockets
-julia> varinfo(WebSockets)
-help?> serve
-help?> WebSockets.open
-julia> cd(joinpath((WebSockets |> Base.pathof |> splitdir)[1],  "..", "examples"))
-julia> readdir()
-julia> include("chat_explore.jl")
+julia> # define what to do with http requests, and with websocket upgrades.
+julia> serverWS =  ServerWS((r) -> WebSockets.Response(200, "OK"),
+                            (ws_server) -> (writeguarded(ws_server, "Hello");
+                                                readguarded(ws_server)));
+julia> # serve on socket 8000, but in a coroutine so we can do other things too.
+julia> @async WebSockets.serve(serverWS, 8000)
+julia> # We ask for a http response, now as our alter ego the client.
+julia> WebSockets.HTTP.get("http://127.0.0.1:8000")
+julia> # Talk to ourselves! Print the first response in blue, then hang up.
+julia> WebSockets.open("ws://127.0.0.1:8000") do ws_client
+           data, success = readguarded(ws_client)
+           if success
+               printstyled(color=:blue, String(data))
+           end
+       end
+julia> # Tell ourselves, the server in a different coroutine: we can stop listening now.
+julia> put!(serverWS.in, "x")
 ```
-### Open a client side connection
-Client side websockets are created by calling `WebSockets.open` (with a server running somewhere). Example (you can run this in a second REPL, or in the same):
-```julia
-julia> cd(joinpath((WebSockets |> Base.pathof |> splitdir)[1],  "..", "examples"))
-julia> include("client_repl_input.jl")
-```
-We recommend `readguarded` and `writeguarded` instead of `read`and `write` for more effective debugging.
-
-### Debugging server side connections
-
-Server side websockets are asyncronous [tasks](https://docs.julialang.org/en/stable/stdlib/parallel/#Tasks-1), which makes debugging harder. The error messages may not spill into the REPL. There are two interfaces to starting a server which includes a websocket handling function:
+More things to do: Access inline documentation and have a look at the examples folder. The testing files demonstrate a variety of uses. Benchmarks show examples of websockets and servers running on separate processes, as oposed to asyncronous tasks.
 
-##### Using WebSockets.serve
-Error messages are directed to a channel. See inline docs: ?Websockets.serve.
+### About this package
+Originally from 2013 and Julia 0.2, the WebSockets API has remained largely unchanged. It now depends on [HTTP.jl](https://github.com/JuliaWeb/HTTP.jl) for establishing the http connections. That package is in ambitious development, and most functionality of this package is already implemented directly in HTTP.jl.
 
-##### Using HTTP.listen
-Error messages are by default sent as messages to the client. This is not good practice if you're serving pages to the internet, but nice while developing locally.
+This more downstream package may lag behind the latest version of HTTP.jl, and in so doing perhaps avoid some borderline bugs. This is why the examples and tests do not import HTTP methods directly, but rely on the methods imported in this package. E.g. by using `WebSockets.HTTP.listen` instead of `HTTP.listen` you may possibly be using the previous release of package HTTP. The imported HTTP version is capped so as to avoid possible issues when new versions of HTTP are released.
 
-## What is nice with WebSockets.jl?
-Some packages rely on WebSockets for communication. You can also use it directly:
+## What can you do with it?
+- read and write between entities you can program or know about
+- serve an svg file to the web browser, containing javascript for connecting back through a websocket, adding two-way interaction with graphics
+- enjoy very low latency and high speed with a minimum of edge case coding
+- implement your own 'if X send this, Y do that' subprotocols. Typically,
+  one subprotocol for sensor input, another for graphics or text to a display.
+- use registered [websocket subprotocols](https://www.iana.org/assignments/websocket/websocket.xml#version-number) for e.g. remote controlled hardware
+- relay user interaction to backend simulations
+- build a network including browser clients and long-running relay servers
+- use convenience functions for gatekeeping
 
-- reading and writing between entities you can program or know about
-- low latency, high speed messaging
-- implementing your own 'if X send this, Y do that' subprotocols
-- registered [websocket subprotocols](https://www.iana.org/assignments/websocket/websocket.xml#version-number) for e.g. remote controlled hardware
-- heartbeating, relaying user interaction to backend simulations
-- build a network including browser clients
-- convenience functions for gatekeeping
-- putting http handlers and websocket coroutines ('handlers') in the same process can be a security advantage. It is good practice to modify web page responses to include time-limited tokens in the wsuri.
+WebSockets are well suited for user interactions via a browser or [cross-platform applications](https://electronjs.org/) like electron. Workload and development time can be moved off Julia resources, error checking code can be reduced. Preferably use websockets for passing arguments, not code, between compiled functions on both sides; it has both speed and security advantages over passing code for evaluation.
 
-WebSockets are well suited for user interactions via a browser or [cross-platform applications](https://electronjs.org/) like electron. Workload and development time can be moved off Julia resources, error checking code can be reduced. Use websockets to pass arguments between compiled functions on both sides; it has both speed and security advantages over passing code for evaluation.
+## Other tips
+- putting http handlers and websocket coroutines ('handlers') in the same process can be a security advantage. It is good practice to modify web page responses to include time-limited tokens in the address, the wsuri.
+- Since `read` and `readguared` are blocking functions, you can easily end up reading indefinitely from any side of the connection. See the `close` function code for an example of non-blocking read with a timeout.
+- Compression is not currenlty implemented, but easily adaptable. On local connections, there's probably not much to gain.
+- If you worry about milliseconds, TCP quirks like 'warm-up' time with low transmission speed after a pause can be avoided with heartbeats. High-performance examples are missing.
+- Garbage collection increases message latency at semi-random intervals, as is visible in  benchmark plots. Benchmarks should include non-memory-allocating examples.
 
-The /logutils folder contains some specialized logging functionality that is quite fast and can make working with multiple asyncronous tasks easier. See /benchmark code for how to use. Logging  may be moved entirely out of WebSockets.jl in the future.
+##### Debugging with WebSockets.ServeWS servers
+Error messages from run-time are directed to a .out channel. See inline docs: ?Websockets.serve.
 
-You can also have a look at alternative Julia packages: [DandelionWebSockets](https://github.com/dandeliondeathray/DandelionWebSockets.jl) or the implementation currently part of [HTTP.jl](https://github.com/JuliaWeb/HTTP.jl).
+##### Debugging with WebSockets.HTTP.listen servers
+Error messages may be sent as messages to the client. This may not be good practice if you're serving pages to the internet, but nice while developing locally. There are some inline comments in the source code which may be of help.
 
-## What are the main downsides to using WebSockets.jl directly?
+## Further development and comments
+The issues section is used for planning development: Contributions are welcome.
 
-- Logging. We need customizable and very fast logging for building networked applications.
-- Compression is not implemented.
-- Possibly non-compliant proxies on the internet, company firewalls.
-- 'Warm-up', i.e. compilation when a method is first used. Warm-up is excluded from current benchmarks.
-- Garbage collection, which increases message latency at semi-random intervals. See benchmark plots.
-- If a connection is closed improperly, the connection task will throw uncaught ECONNRESET and similar messages.
-- TCP quirks, including 'warm-up' time with low transmission speed after a pause. Heartbeats can alleviate.
-- Since `read` is a blocking function, you can easily end up reading indefinitely from any side of the connection. See the `close function code for an example of non-blocking read with a timeout.
+- The /logutils and /benchmark folders contain some features that are not currently fully implemented (or working?), namely a specialized logger. For application development, we generally require very fast logging and this approach may or may not be sufficiently fast.
+- Alternative Julia packages: [DandelionWebSockets](https://github.com/dandeliondeathray/DandelionWebSockets.jl) and the direct implementation in [HTTP.jl](https://github.com/JuliaWeb/HTTP.jl).
 
 ## Errors after updating?
+### To version 1.1.0
+This version is driven by large restructuring in HTTP.jl. We import more functions and types into WebSockets, e.g., WebSockets.Request. The main interface does not, intentionally, change, except for 'origin', which should now be qualified as WebSockets.origin.
 
+### To version 0.5.0
 The introduction of client side websockets to this package in version 0.5.0 may require changes in your code:
 - The `WebSocket.id` field is no longer supported. You can generate unique counters by code similar to 'bencmark/functions_open_browsers.jl' COUNTBROWSER.
 - You may want to modify you error handling code. Examine WebSocketsClosedError.message.

REQUIRE

@@ -1,4 +1,3 @@
-julia 0.7
+julia 0.7 1.99
 MbedTLS
-HTTP 0.6.14 0.6.15
-
+HTTP 0.7.0 0.7.99

appveyor.yml

@@ -10,9 +10,9 @@ platform:
 
 # # Uncomment the following lines to allow failures on nightly julia
 # # (tests will run but not make your overall status red)
-# matrix:
-#   allow_failures:
-#   - julia_version: latest
+matrix:
+  allow_failures:
+  - julia_version: latest
 
 branches:
   only:
@@ -40,4 +40,4 @@ test_script:
 # # which would have coverage gaps without running on Windows
 # on_success:
 #   - echo "%JL_CODECOV_SCRIPT%"
-#   - C:\julia\bin\julia -e "%JL_CODECOV_SCRIPT%"
+#   - C:\julia\bin\julia -e "%JL_CODECOV_SCRIPT%"

benchmark/ws_hts.jl

@@ -17,7 +17,7 @@ const PORT = 8000
 const SERVER = "127.0.0.1"
 const WSMAXTIME = Base.Dates.Second(600)
 const WEBSOCKET = Vector{WebSockets.WebSocket}()
-const TCPREF = Ref{HTTP.Sockets.TCPServer}()
+const TCPREF = Ref{Base.IOServer}()
 "Run asyncronously or in separate process"
 function listen_hts()
     id = "listen_hts"
@@ -29,7 +29,7 @@ function listen_hts()
                 acceptholdws(http)
                 clog(id, "Websocket closed, server stays open until ws_hts.close_hts()")
             else
-                HTTP.Servers.handle_request(handlerequest, http)
+                WebSockets.handle_request(handlerequest, http)
             end
         end
     catch err
@@ -165,4 +165,4 @@ import ws_hts.listen_hts
 tas = @async ws_hts.listen_hts()
 sleep(7)
 hts = ws_hts.getws_hts()
-"""
+"""

examples/chat_explore.jl

@@ -1,48 +1,42 @@
-#=
-
-A chat server application. Starts a new task for each browser (tab) that connects.
-
-To use:
-    - include("chat_explore.jl") in REPL
-    - start a browser on the local ip address, e.g.: http://192.168.0.4:8080
-    - inspect global variables starting with 'last' while the chat is running asyncronously
-
-To call in from other devices, figure out your IP address on the network and change the 'gatekeeper' code.
-
-Functions used as arguments are explicitly defined with names instead of anonymous functions (do..end constructs).
-This may improve debugging readability at the cost of increased verbosity.
+using WebSockets
+import WebSockets:Response,
+                  Request
+using Dates
+using Sockets
 
-=#
 global lastreq = 0
 global lastws = 0
 global lastmsg = 0
 global lastws = 0
 global lastserver = 0
 
-using WebSockets
-import WebSockets:Response,
-                  Request,
-                  HandlerFunction,
-                  WebsocketHandler
-using Dates
-import Sockets
 const CLOSEAFTER = Dates.Second(1800)
 const HTTPPORT = 8080
 const LOCALIP = string(Sockets.getipaddr())
 const USERNAMES = Dict{String, WebSocket}()
 const HTMLSTRING = read(joinpath(@__DIR__, "chat_explore.html"), String)
 
 
+@info """
+A chat server application. For each browser (tab) that connects,
+an 'asyncronous function' aka 'coroutine' aka 'task' is started.
+
+To use:
+    - include("chat_explore.jl") in REPL
+    - start a browser on the local ip address, e.g.: http://192.168.0.4:8080
+    - inspect global variables starting with 'last' while the chat is running asyncronously
+
+"""
+
 # Since we are to access a websocket from outside
 # it's own websocket handler coroutine, we need some kind of
 # mutable container for storing references:
 const WEBSOCKETS = Dict{WebSocket, Int}()
 
 """
-Called by 'gatekeeper', this function stays active while the
+Called by 'gatekeeper', this function will be running in a task while the
 particular websocket is open. The argument is an open websocket.
-Other instances of the function run in other tasks. The tasks
-are started by HTTP.
+Other instances of the function run in other tasks.
 """
 function coroutine(thisws)
     global lastws = thisws
@@ -138,19 +132,37 @@ end
 "Request to response. Response is the predefined HTML page with some javascript"
 req2resp(req::Request) = HTMLSTRING |> Response
 
-# The server takes two function wrappers; one handler function for page requests,
-# one for opening websockets (which the javascript in the HTML page will try to do)
-global lastserver = WebSockets.ServerWS(HandlerFunction(req2resp), WebsocketHandler(gatekeeper))
+
+# The following lines disblle detail messages from spilling into the
+# REPL. Remove the it to gain insight.
+using Logging
+import Logging.shouldlog
+function shouldlog(::ConsoleLogger, level, _module, group, id)
+    if _module == WebSockets.HTTP.Servers
+        if level == Logging.Warn || level == Logging.Info
+            return false
+        else
+            return true
+        end
+    else
+        return true
+    end
+end
+
+
+# ServerWS takes two functions; the first a http request handler function for page requests,
+# one for opening websockets (which javascript in the HTML page will try to do)
+global lastserver = WebSockets.ServerWS(req2resp, gatekeeper)
 
 # Start the server asyncronously, and stop it later
-global litas = @async WebSockets.serve(lastserver, LOCALIP, HTTPPORT)
+@async WebSockets.serve(lastserver, LOCALIP, HTTPPORT)
 @async begin
     println("HTTP server listening on $LOCALIP:$HTTPPORT for $CLOSEAFTER")
     sleep(CLOSEAFTER.value)
     println("Time out, closing down $HTTPPORT")
-    Base.throwto(litas, InterruptException())
-    # Alternative close method: see ?WebSockets.serve
+    put!(lastserver.in, "I can send anything, you close")
+    nothing
 end
 
 
-nothing
+nothing

examples/minimal_server.jl

@@ -36,7 +36,7 @@ function gatekeeper(req, ws)
     end
 end
 
-const handle(req) = replace(BAREHTML, "<body></body>" => BODY) |> WebSockets.Response
+handle(req) = replace(BAREHTML, "<body></body>" => BODY) |> WebSockets.Response
 
 const server = WebSockets.ServerWS(handle,
                                     gatekeeper)

examples/minimal_server_listen_do_syntax.jl

@@ -3,8 +3,7 @@ const BAREHTML = "<head><meta http-equiv=\"Content-Type\" content=\"text/html; c
  <title>Empty.html</title></head><body></body></html>"
 import Sockets
 using WebSockets
-import HTTP.listen
-import HTTP.Servers.handle_request
+import WebSockets.handle_request
 const LOCALIP = string(Sockets.getipaddr())
 const PORT = 8080
 const BODY =  "<body><p>Press F12
@@ -13,18 +12,15 @@ const BODY =  "<body><p>Press F12
                 <p>ws.send(\"Browser console says hello!\")
                 </body>"
 
-global TCPREF = Ref{Base.IOServer}()
+const SERVERREF = Ref{Base.IOServer}()
 @info("Browser > $LOCALIP:$PORT , F12> console > ws = new WebSocket(\"ws://$LOCALIP:$PORT\") ")
 try
-    listen(LOCALIP, UInt16(PORT), tcpref = TCPREF) do http
-        if WebSockets.is_upgrade(http.message)
-            WebSockets.upgrade(http) do req, ws
+    WebSockets.HTTP.listen(LOCALIP, UInt16(PORT), tcpref = SERVERREF) do stream
+        if WebSockets.is_upgrade(stream.message)
+            WebSockets.upgrade(stream) do req, ws
                 orig = WebSockets.origin(req)
                 println("\nOrigin:", orig, "    Target:", target(req), "    subprotocol:", subprotocol(req))
                 if occursin(LOCALIP, orig)
-                    coroutine(ws)
-                elseif orig == ""
-                    @info "Non-browser clients don't send Origin. We liberally accept the update request in this case."
                     while isopen(ws)
                         data, = readguarded(ws)
                         s = String(data)
@@ -35,12 +31,14 @@ try
                         println("Received: ", s)
                         writeguarded(ws, "Hello! Send empty message to exit, or just leave.")
                     end
+                elseif orig == ""
+                    @info "Nice try. But this example only accepts browser connections."
                 else
                     @warn "Inacceptable request"
                 end
             end
         else
-            handle_request(http) do req
+            handle_request(stream) do req
                 replace(BAREHTML, "<body></body>" => BODY) |> WebSockets.Response
             end
         end