diff --git a/._webroot b/._webroot
new file mode 100644
index 0000000..0d8176a
Binary files /dev/null and b/._webroot differ
diff --git a/README.md b/README.md
index 616ce1c..64885f0 100644
--- a/README.md
+++ b/README.md
@@ -1,8 +1,24 @@
# code_401_wk2_http
Code 401 week 2 HTTP server project
+Step 1:
+ - open listening socket using socket library
+ - server loop to take incoming requests and send them back
+ - client loop to receive requests and return them to console
+
+
Step 2:
+ - response_ok function to send HTTP 200 code response
+ - response_error function to send HTTP 500 server error response
+
+
+Step 3:
- added request validation
- added error codes for bad requests
- extract URI from valid response.
- return appropriate HTTP response to client
+
+
+Step Final:
+ - added gevent to support server concurrency
+ - modified tests to be OS independent (in parallel with step 3)
diff --git a/setup.py b/setup.py
index 9b0d089..7699ddd 100644
--- a/setup.py
+++ b/setup.py
@@ -12,7 +12,7 @@
author_email='weatherford.william@gmail.com',
license='MIT',
packages=[], # all your python packages with an __init__ file
- py_modules=['server', 'client'], # your python modules to include
+ py_modules=['server', 'client', 'con_server'],
package_dir={'': 'src'},
install_requires=[],
extras_require={'test': ['pytest', 'pytest-xdist', 'tox']}
diff --git a/src/client.py b/src/client.py
index 9967728..d8cc2b0 100644
--- a/src/client.py
+++ b/src/client.py
@@ -14,14 +14,15 @@ def client(msg):
cli_sock = socket.socket(*stream_info[:3])
cli_sock.connect(stream_info[-1])
- cli_sock.sendall(msg.encode('utf8'))
+ cli_sock.sendall(msg.encode('utf-8'))
cli_sock.shutdown(socket.SHUT_WR)
- response = ''
+ response_parts = []
while True:
part = cli_sock.recv(BUFFER_LENGTH)
- response += part.decode('utf-8')
+ response_parts.append(part)
if len(part) < BUFFER_LENGTH:
break
+ response = b''.join(response_parts).decode('utf-8')
cli_sock.close()
return response
diff --git a/src/con_server.py b/src/con_server.py
new file mode 100644
index 0000000..7f13a15
--- /dev/null
+++ b/src/con_server.py
@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+"""Gevent powered concurrency server."""
+import sys
+
+
+if __name__ == '__main__':
+ from gevent.server import StreamServer
+ from gevent.monkey import patch_all
+ from server import http_server
+ patch_all()
+ server = StreamServer(('127.0.0.1', 5000), http_server)
+ print('Starting http server on port 5000')
+ try:
+ server.serve_forever()
+ except Exception as e:
+ print(e.msg)
+ finally:
+ print('\nShutting down the server...\n')
+ server.close()
+ sys.exit()
diff --git a/src/server.py b/src/server.py
index a15730d..89f3df6 100644
--- a/src/server.py
+++ b/src/server.py
@@ -1,24 +1,59 @@
# -*- coding: utf-8 -*-
"""Server module to to take HTTP requests."""
+import io
+import mimetypes
+import os
import socket
import sys
import time
+from contextlib import closing
-OK_200 = b'200 OK'
-ERR_400 = b'400: Bad Request'
-ERR_405 = b'405: Method Not Allowed'
-ERR_505 = b'505: HTTP Version Not Supported'
BUFFER_LENGTH = 8
ADDRINFO = ('127.0.0.1', 5000)
-
-# TODO: update response assemble and tests for more sophisticated
-# testing of each component. Assert for required parts while disrgarding
-# optional components (but making sure they are in the right place).
-# split on \r\n\r\n assert there are 2 pieces
-# for first line of response, use maxsplit arg to split
-# e.g. "HTTP/1.1 500 Internal Server Error".split(' ', maxsplit=2)
-# >>> ["HTTP/1.1", "500", "Internal Server Error"]
+PARENT_DIR = os.path.dirname(__file__)
+GRANDPARENT_DIR = os.path.join(PARENT_DIR, '..')
+WEBROOT_PATH = os.path.join(GRANDPARENT_DIR, 'webroot', '')
+
+# Unicode constants for standard re-usable parts.
+CRLF = '\r\n'
+GET = 'GET'
+HTTP1_1 = 'HTTP/1.1'
+HOST = 'Host: '
+CONTENT_TYPE = 'Content-Type: {}'
+CONTENT_LENGTH = 'Content-Length: {}'
+
+TEXT_HTML = 'text/html; charset=utf-8'
+TEXT_PLAIN = 'text/plain; charset=utf-8'
+
+# Easy Reference dictionary with int keys for code/message part.
+HTTP_CODES = {
+ 200: '200 OK',
+ 400: '400 Bad Request',
+ 404: '404 Not Found',
+ 405: '405 Method Not Allowed',
+ 505: '505 HTTP Version Not Supported',
+}
+
+# updated HTML templates to use unicode strings
+HTML_TEMPLATE = '''
+
+
+
+
+
+
+
+ {header}
+
+
+
+ '''
+HTML_LI_TEMPLATE = '''
+ {short_path}
+ '''
def server():
@@ -33,94 +68,155 @@ def server():
try:
while True:
conn, addr = serv_sock.accept()
- request = []
- while True:
- part = conn.recv(BUFFER_LENGTH)
- request.append(part)
- # conn.sendall(part)
- if len(part) < BUFFER_LENGTH:
- break
- request = b''.join(request)
- print('Request received:')
- print(request.decode('utf-8'))
- try:
- uri = parse_request(request)
- response = response_ok()
- except ValueError as e:
- response = response_error(*e.args)
- conn.sendall(response)
- conn.close()
- time.sleep(0.01)
- except KeyboardInterrupt:
- pass
+ http_server(conn, addr)
+ except Exception as e:
+ print(e.msg)
finally:
- try:
- conn.close()
- except NameError:
- pass
print('\nShutting down the server...\n')
serv_sock.close()
sys.exit()
-def parse_request(request):
- """TOOD: Docstring."""
- # only accept GET
- # only HTTP/1.1
- # validate that proper Host: header was specified
- # other requests raise appropriate Python error
- # if no conditions arise, should return the URI
- method = b''
- uri = b''
- protocol = b''
- headers = b''
+def http_server(conn, addr):
+ """Take HTTP requests and return appropriate HTTP response."""
+ with closing(conn):
+ request_parts = []
+ while True:
+ part = conn.recv(BUFFER_LENGTH)
+ request_parts.append(part)
+ if len(part) < BUFFER_LENGTH:
+ break
+ # Immediately decode all of incoming request into unicode.
+ # In future, may need to check Content-type of incoming request?
+ request = b''.join(request_parts).decode('utf-8')
+ print('Request received:\n{}'.format(request))
+ try:
+ uri = parse_request(request)
+ # Here body might be a bytestring.
+ body, content_type = resolve_uri(uri)
+ body_length = len(body)
+ response_headers = response_ok(content_type, body_length)
- try:
- headers, body = request.split(b'\r\n\r\n')
- except ValueError:
- raise ValueError(ERR_400)
+ except ValueError as e:
+ err_code = e.args[0]
+ response_headers = response_error(err_code)
+ body = HTTP_CODES[err_code].encode('utf-8')
+
+ # Re-encode into bytes on the way out.
+ response_headers = response_headers.encode('utf-8')
+
+ response = b''.join([response_headers, body])
+
+ conn.sendall(response)
+ conn.close()
+ time.sleep(0.01)
- try:
- headers = headers.split(b'\r\n')
- first_line = headers[0]
- except (ValueError, IndexError):
- raise ValueError(ERR_400)
+def parse_request(request):
+ """Parse client request."""
+ method = ''
+ uri = ''
+ protocol = ''
+ headers = ''
+
+ # I was able to simplify this into one try/except block.
+ # The key was to organize it by error type.
try:
+ headers, body = request.split(CRLF * 2)
+ headers = headers.split(CRLF)
+ try:
+ first_line = headers[0]
+ except IndexError:
+ raise ValueError(400)
method, uri, protocol = first_line.split()
+ headers = headers[1:]
+ for h in headers:
+ if h.startswith(HOST):
+ break
+ else:
+ raise ValueError(400)
except ValueError:
- raise ValueError(ERR_400)
+ raise ValueError(400)
+ if method != GET:
+ raise ValueError(405)
+ if protocol != HTTP1_1:
+ raise ValueError(505)
+
+ return uri
+
+
+def resolve_uri(uri):
+ """Return tuple of content and content type, or raise 404 error."""
+ print('Requested URI: ', uri)
+ uri = full_uri(uri)
+ print('URI after join: ', uri)
- headers = headers[1:]
- for h in headers:
- if h.startswith(b'Host: '):
- break
+ if os.path.isfile(uri):
+ content_type = mimetypes.guess_type(uri)[0]
+ body = read_file_bytes(uri)
+
+ elif os.path.isdir(uri):
+ content_type = TEXT_HTML
+ body = display(next(os.walk(uri)))
+ body = body.encode('utf-8')
else:
- raise ValueError(ERR_400)
+ print(uri, 'is not a file or dir.')
+ raise ValueError(404)
+ return (body, content_type)
- if method != b'GET':
- raise ValueError(ERR_405)
- if protocol != b'HTTP/1.1':
- raise ValueError(ERR_505)
+def full_uri(uri):
+ """Take a unicode uri from webroot and return the absolute path."""
+ uri = os.path.join(*uri.split('/'))
+ uri = os.path.join(WEBROOT_PATH, uri)
return uri
-def response_ok():
+def web_uri(uri):
+ """Take a unicode uri and return the uri from webroot."""
+ return uri.replace(WEBROOT_PATH, '')
+
+
+def read_file_bytes(path):
+ """Return the data in bytestring format from the file at a give path."""
+ f = io.open(path, 'rb')
+ data = f.read()
+ f.close()
+ return data
+
+
+# Response_ok doesn't parse body (might be bytes as for an image).
+# References key 200 from HTTP_CODES reference dictionary.
+def response_ok(content_type, body_length):
"""Return 'HTTP/1.1 200 OK' for when connection ok."""
- return (b'HTTP/1.1 %s\r\n'
- b'Content-Type: text/plain\r\n'
- b'\r\n'
- b'Welcome to Imperial Space, rebel scum.\n'
- b'|-o-| <-o-> |-o-|') % OK_200
+ return CRLF.join([' '.join([HTTP1_1, HTTP_CODES[200]]),
+ CONTENT_TYPE.format(content_type),
+ CONTENT_LENGTH.format(body_length),
+ CRLF])
-def response_error(err_msg):
+# Response_error take an int as err_code, to reference HTTP_CODES dict.
+def response_error(err_code):
"""Return 'Internal Server Error' for when problem occurs."""
- return (b'HTTP/1.1 %s\r\n'
- b'Content-Type: text/plain\r\n'
- b'\r\n'
- b'Death Star Error. Please build again.') % err_msg
+ return CRLF.join([' '.join([HTTP1_1, HTTP_CODES[err_code]]),
+ CONTENT_TYPE.format(TEXT_PLAIN),
+ CRLF])
+
+
+def display(threeple):
+ """Split dir threeple into components and return as HTML."""
+ cur_dir, dir_subdir, dir_files = threeple
+ cur_dir = web_uri(cur_dir)
+ dir_list = []
+ for i in dir_subdir + dir_files:
+ if i.startswith('._'):
+ continue
+ full_path = os.path.join(cur_dir, i)
+ html_li = HTML_LI_TEMPLATE.format(full_path=full_path, short_path=i)
+ dir_list.append(html_li)
+
+ return HTML_TEMPLATE.format(header=cur_dir,
+ list_items=''.join(dir_list))
if __name__ == '__main__':
diff --git a/src/test_server.py b/src/test_server.py
index 89c2cb7..82cc74f 100644
--- a/src/test_server.py
+++ b/src/test_server.py
@@ -1,118 +1,181 @@
# -*- coding: utf-8 -*-
"""Test module for client and server modules."""
+import os
import pytest
-from server import BUFFER_LENGTH, OK_200, ERR_400, ERR_405, ERR_505
+from server import WEBROOT_PATH, CRLF, HTTP1_1, HTTP_CODES
+from server import CONTENT_TYPE, TEXT_HTML
+WEBROOT_STUB = os.path.join('', 'webroot', '')
-U_H = u'HTTP/1.1'
-U_200 = u'{} {}'.format(U_H, OK_200.decode('utf-8'))
-U_400 = u'{} {}'.format(U_H, ERR_400.decode('utf-8'))
-U_405 = u'{} {}'.format(U_H, ERR_405.decode('utf-8'))
-U_505 = u'{} {}'.format(U_H, ERR_505.decode('utf-8'))
+METHODS = [
+ ('GET', 200),
+ ('POST', 405),
+ ('DELETE', 405),
+ ('jslalsijhr;;', 405),
+]
+URIS = [
+ ('/', 200, WEBROOT_PATH),
+ ('/a_web_page.html', 200, os.path.join(WEBROOT_PATH, 'a_web_page.html')),
+ ('/images', 200, os.path.join(WEBROOT_PATH, 'images')),
+ ('/sample.txt', 200, os.path.join(WEBROOT_PATH, 'sample.txt')),
+ ('', 400, WEBROOT_PATH),
+]
-TESTS = [
- 'aaaaaaaaaaaaaaaaaaaaaaa',
- 'aa',
- 'a' * BUFFER_LENGTH,
- u'£©°',
+PROTOS = [
+ (HTTP1_1, 200),
+ ('HTTP/1.0', 505),
+ ('jhdo%#@#4939', 505),
+ ('', 400),
]
-GOOD_REQUEST = (b'GET /index.html HTTP/1.1\r\n'
- b'Host: theempire.com\r\n'
- b'\r\n')
+HEADERS = [
+ ('Host: example.com', 200),
+ ('Host: example.com' + CRLF + CONTENT_TYPE.format(TEXT_HTML), 200),
+ ('Host example.com', 400),
+ ('', 400),
+]
+EMPTY_LINES = [
+ (CRLF, 200),
+ ('p40kdnad', 400),
+ ('', 400),
+]
-BAD_NOT_GET = (b'POST /index.html HTTP/1.1\r\n'
- b'Host: theempire.com\r\n'
- b'\r\n')
-BAD_NO_HOST = (b'GET /index.html HTTP/1.1\r\n'
- b'\r\n')
+BODIES = [
+ ('', 200),
+ ('Some HTML', 200),
+]
-BAD_NO_PROTO = (b'GET /index.html\r\n'
- b'Host: theempire.com\r\n'
- b'\r\n')
-BAD_WRONG_PROTO = (b'GET /index.html HTTP/1.0\r\n'
- b'Host: theempire.com\r\n'
- b'\r\n')
+@pytest.fixture(scope='module', params=METHODS)
+def method(request):
+ """Establish fixtures for the method part of test HTTP requests."""
+ return request.param
-BAD_NO_CRLF = (b'GET /index.html HTTP/1.1\r\n'
- b'Host: theempire.com\r\n')
-U_G_R = u'{}'.format(GOOD_REQUEST.decode('utf-8'))
-U_BNG = u'{}'.format(BAD_NOT_GET.decode('utf-8'))
-U_BNH = u'{}'.format(BAD_NO_HOST.decode('utf-8'))
-U_BNP = u'{}'.format(BAD_NO_PROTO.decode('utf-8'))
-U_BWP = u'{}'.format(BAD_WRONG_PROTO.decode('utf-8'))
-U_BNC = u'{}'.format(BAD_NO_CRLF.decode('utf-8'))
+@pytest.fixture(scope='module', params=URIS)
+def uri(request):
+ """Establish fixtures for the URI part of test HTTP requests."""
+ return request.param
-TEST_PARSE = [
- (GOOD_REQUEST, None, OK_200, b'/index.html'),
- (BAD_NOT_GET, ValueError, ERR_405, b''),
- (BAD_NO_HOST, ValueError, ERR_400, b''),
- (BAD_NO_PROTO, ValueError, ERR_400, b''),
- (BAD_WRONG_PROTO, ValueError, ERR_505, b''),
- (BAD_NO_CRLF, ValueError, ERR_400, b''),
-]
-TEST_CLI_REQUEST = [
- (U_G_R, U_200),
- (U_BNG, U_405),
- (U_BNH, U_400),
- (U_BNP, U_400),
- (U_BWP, U_505),
- (U_BNC, U_400),
-]
+@pytest.fixture(scope='module', params=PROTOS)
+def proto(request):
+ """Establish fixtures for the protocol part of test HTTP requests."""
+ return request.param
-ERR_LIST = [
- ERR_400,
- ERR_405,
- ERR_505,
-]
+@pytest.fixture(scope='module', params=HEADERS)
+def headers(request):
+ """Establish fixtures for the headers part of test HTTP requests."""
+ return request.param
+
+
+@pytest.fixture(scope='module', params=EMPTY_LINES)
+def empty_line(request):
+ """Establish fixtures for the empty line in test HTTP requests."""
+ return request.param
+
+
+@pytest.fixture(scope='module', params=BODIES)
+def body(request):
+ """Establish fixtures for the content body of test HTTP requests."""
+ return request.param
-# @pytest.mark.parametrize('msg', TESTS)
-# def test_system(msg):
-# """Test that messages to server are returned as the same message."""
-# from client import client
-# assert client(msg) == msg
+@pytest.fixture(scope='module')
+def make_request(method, uri, proto, headers, empty_line, body):
+ """Create many different requests to check for correct response."""
+ expected_code = 200
+ error = None
+ # Our parse request finds the code in this order: 400, 405, 505, 200
+ # All possible error codes associated with all parts of the request.
+ poss_err_codes = [part[1] for part in
+ [method, uri, proto, headers, empty_line, body]
+ if part[1] in ERR_CODES]
+ if poss_err_codes:
+ expected_code = min(poss_err_codes)
+ error = ValueError(expected_code)
-@pytest.mark.parametrize('cli_request, msg', TEST_CLI_REQUEST)
-def test_system(cli_request, msg):
- """Test that messages to server are returned as the same message."""
+ top_line = ' '.join([part[0] for part in [method, uri, proto]])
+ rest = CRLF.join([part[0] for part in [headers, empty_line]])
+ request = CRLF.join([top_line, rest]) + body[0]
+ return (request, expected_code, error, uri[0])
+
+
+ERR_CODES = [n for n in HTTP_CODES.keys() if n >= 400]
+
+
+SAMPLE_TXT = os.linesep.join(['This is a very simple text file.',
+ 'Just to show that we can serve it up.',
+ 'It is three lines long.',
+ ''])
+
+
+def test_system(make_request):
+ """Test that messages send to the server get appropriate response."""
from client import client
- response = client(cli_request)
- response_parts = response.split('\r\n')
- assert response_parts[0] == msg
+ request, code, error, uri_response = make_request
+ response = client(request)
+ response_parts = response.split(CRLF)
+ assert response_parts[0] == ' '.join([HTTP1_1, HTTP_CODES[code]])
assert '' in response_parts
-@pytest.mark.parametrize('cli_request, error, msg, uri', TEST_PARSE)
-def test_parse_request(cli_request, error, msg, uri):
+def test_parse_request(make_request):
"""Test that parse_request returns the URI or raises appropriate error."""
from server import parse_request
+ request, code, error, uri_response = make_request
if error:
- with pytest.raises(error) as e:
- parse_request(cli_request)
- assert e.args[0] == msg
+ try:
+ parse_request(request)
+ assert False # If test reaches here, it has failed to raise error.
+ except ValueError as e:
+ assert e.args[0] == code
else:
- assert parse_request(cli_request) == uri
+ assert parse_request(request) == uri_response
def test_response_ok():
"""Test that response_ok returns '200 OK' if connection is good."""
from server import response_ok
- assert response_ok().split(b'\r\n')[0] == b'HTTP/1.1 %s' % OK_200
+ response = response_ok('text/html', 0)
+ first_line = response.split(CRLF)[0]
+ assert first_line == ' '.join((HTTP1_1, HTTP_CODES[200]))
-@pytest.mark.parametrize('err_msg', ERR_LIST)
-def test_response_error(err_msg):
+@pytest.mark.parametrize('err_code', ERR_CODES)
+def test_response_error(err_code):
"""Test that response_error returns '500 Internal Server Error'."""
from server import response_error
- error_text = b'HTTP/1.1 %s' % err_msg
- assert response_error(err_msg).split(b'\r\n')[0] == error_text
+ response = response_error(err_code)
+ first_line = response.split(CRLF)[0]
+ assert first_line == ' '.join((HTTP1_1, HTTP_CODES[err_code]))
+
+
+def test_join_uri(uri):
+ """Test that webroot is accessible."""
+ from server import full_uri
+ uri_in, code, uri_out = uri
+ assert full_uri(uri_in) == uri_out
+
+
+def test_resolve_uri(uri):
+ """Test that resolve_uri returns content and content-type."""
+ from server import resolve_uri
+ uri_in, code, uri_out = uri
+ body, content_type = resolve_uri(uri_in)
+ assert isinstance(body, bytes)
+
+
+def test_webroot():
+ """Test that webroot is accessible."""
+ from server import WEBROOT_PATH, read_file_bytes
+ import os
+ sample_path = os.path.join(WEBROOT_PATH, 'sample.txt')
+ data = read_file_bytes(sample_path)
+ assert data.decode('utf-8') == SAMPLE_TXT
diff --git a/webroot/._a_web_page.html b/webroot/._a_web_page.html
new file mode 100644
index 0000000..0d8176a
Binary files /dev/null and b/webroot/._a_web_page.html differ
diff --git a/webroot/._images b/webroot/._images
new file mode 100644
index 0000000..0d8176a
Binary files /dev/null and b/webroot/._images differ
diff --git a/webroot/._make_time.py b/webroot/._make_time.py
new file mode 100644
index 0000000..0d8176a
Binary files /dev/null and b/webroot/._make_time.py differ
diff --git a/webroot/._sample.txt b/webroot/._sample.txt
new file mode 100644
index 0000000..0d8176a
Binary files /dev/null and b/webroot/._sample.txt differ
diff --git a/webroot/a_web_page.html b/webroot/a_web_page.html
new file mode 100644
index 0000000..edf85fd
--- /dev/null
+++ b/webroot/a_web_page.html
@@ -0,0 +1,11 @@
+
+
+
+
+Code Fellows
+
+A fine place to learn Python web programming!
+
+
+
+
diff --git a/webroot/images/._JPEG_example.jpg b/webroot/images/._JPEG_example.jpg
new file mode 100644
index 0000000..0d8176a
Binary files /dev/null and b/webroot/images/._JPEG_example.jpg differ
diff --git a/webroot/images/._Sample_Scene_Balls.jpg b/webroot/images/._Sample_Scene_Balls.jpg
new file mode 100644
index 0000000..0d8176a
Binary files /dev/null and b/webroot/images/._Sample_Scene_Balls.jpg differ
diff --git a/webroot/images/._sample_1.png b/webroot/images/._sample_1.png
new file mode 100644
index 0000000..0d8176a
Binary files /dev/null and b/webroot/images/._sample_1.png differ
diff --git a/webroot/images/JPEG_example.jpg b/webroot/images/JPEG_example.jpg
new file mode 100644
index 0000000..13506f0
Binary files /dev/null and b/webroot/images/JPEG_example.jpg differ
diff --git a/webroot/images/Sample_Scene_Balls.jpg b/webroot/images/Sample_Scene_Balls.jpg
new file mode 100644
index 0000000..1c0ccad
Binary files /dev/null and b/webroot/images/Sample_Scene_Balls.jpg differ
diff --git a/webroot/images/sample_1.png b/webroot/images/sample_1.png
new file mode 100644
index 0000000..5b2f52d
Binary files /dev/null and b/webroot/images/sample_1.png differ
diff --git a/webroot/make_time.py b/webroot/make_time.py
new file mode 100644
index 0000000..b69acf3
--- /dev/null
+++ b/webroot/make_time.py
@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+
+"""
+make_time.py
+
+simple script that returns and HTML page with the current time
+"""
+
+import datetime
+
+time_str = datetime.datetime.now().isoformat()
+
+html = """
+
+
+ The time is:
+ %s
+
+
+""" % time_str
+
+print(html)
diff --git a/webroot/sample.txt b/webroot/sample.txt
new file mode 100644
index 0000000..1b7935d
--- /dev/null
+++ b/webroot/sample.txt
@@ -0,0 +1,3 @@
+This is a very simple text file.
+Just to show that we can serve it up.
+It is three lines long.