diff --git a/MANIFEST.in b/MANIFEST.in index 6599213..2f2eec3 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,4 @@ include README.rst include MANIFEST.in +recursive-include tests *.py prune *.pyc \ No newline at end of file diff --git a/README.rst b/README.rst index e5b6471..cb15c2e 100644 --- a/README.rst +++ b/README.rst @@ -7,7 +7,7 @@ Welcome to Html2Data Description =========== -A simple way to transform a HTML file or URL to structured data. You only need to define the xpath to the element. Optionaly you can define functions to be applied after. You can easily write XPATH using the firebug extension, copy XPATH (I recommend edit the XPATH given by firebug, making it shorter). +A simple way to transform a HTML file or URL to structured data. You only need to define the xpath to the element. Optionally you can define functions to be applied after. You can easily write XPATH using the firebug extension, copy XPATH (I recommend edit the XPATH given by firebug, making it shorter). Installation ============ diff --git a/html2data/__init__.py b/html2data/__init__.py index bdb8cf6..5ceb8e9 100644 --- a/html2data/__init__.py +++ b/html2data/__init__.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- from lxml import etree -from StringIO import StringIO from copy import copy from httplib2 import Http @@ -11,8 +10,8 @@ def __init__(self, html = None, url = None, tree = None): raise Exception('html or url or tree parameters are required') if url: connection = Http() - header, html = connection.request(url) - elif html: + _, html = connection.request(url) + if html: self.tree = self._get_tree_from_html(html) else: self.tree = tree @@ -55,7 +54,6 @@ def _get_one(elements): return (elements or [None])[0] def _apply_after(self, value, apply_after, multiple, strip, text): - total_added_after = 0 if not multiple: apply_after.insert(0, self._get_one) if strip and text: @@ -66,7 +64,9 @@ def _apply_after(self, value, apply_after, multiple, strip, text): value = after(value) return value - def parse_one(self, xpath = None, css = None, multiple = False, apply_after = [], text = True, strip = True): + def parse_one(self, xpath = None, css = None, multiple = False, apply_after = None, text = True, strip = True): + if apply_after is None: + apply_after = [] #TODO: Be able to return elements and text if xpath: value = self.xpath(xpath.replace('/text()', '')) diff --git a/setup.py b/setup.py index 9bfe4fa..528ba55 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ def read(fname): license = "BSD", keywords = "html2data html data xpath crawler transform", url = "https://github.com/dperezrada/html2data", - packages=['html2data', 'tests'], + packages=['html2data', ], long_description=read('README.rst'), include_package_data=True, classifiers=[ diff --git a/tests/test_parse.py b/tests/test_parse.py index 54141e7..90e9bb8 100644 --- a/tests/test_parse.py +++ b/tests/test_parse.py @@ -1,5 +1,4 @@ # encoding: utf-8 -import os from unittest import TestCase diff --git a/tests/test_tree.py b/tests/test_tree.py index 8512d8e..d89895d 100644 --- a/tests/test_tree.py +++ b/tests/test_tree.py @@ -1,8 +1,6 @@ # encoding: utf-8 -import os from unittest import TestCase -from httplib2 import Http from ludibrio import Mock from html2data import HTML2Data