Merge pull request #4 from ceerRep/master

cR's Homework

Author: WhiteCatFly

week_1/2018202133HPY/.gitignore

@@ -0,0 +1,2 @@
+main
+download/*

week_1/2018202133HPY/.vscode/settings.json

@@ -0,0 +1,51 @@
+{
+    "files.associations": {
+        "mutex": "cpp",
+        "array": "cpp",
+        "*.tcc": "cpp",
+        "cctype": "cpp",
+        "chrono": "cpp",
+        "clocale": "cpp",
+        "cmath": "cpp",
+        "condition_variable": "cpp",
+        "cstdint": "cpp",
+        "cstdio": "cpp",
+        "cstdlib": "cpp",
+        "ctime": "cpp",
+        "cwchar": "cpp",
+        "cwctype": "cpp",
+        "unordered_map": "cpp",
+        "vector": "cpp",
+        "exception": "cpp",
+        "fstream": "cpp",
+        "functional": "cpp",
+        "initializer_list": "cpp",
+        "iosfwd": "cpp",
+        "iostream": "cpp",
+        "istream": "cpp",
+        "limits": "cpp",
+        "memory": "cpp",
+        "new": "cpp",
+        "optional": "cpp",
+        "ostream": "cpp",
+        "ratio": "cpp",
+        "sstream": "cpp",
+        "stdexcept": "cpp",
+        "streambuf": "cpp",
+        "string_view": "cpp",
+        "system_error": "cpp",
+        "thread": "cpp",
+        "type_traits": "cpp",
+        "tuple": "cpp",
+        "typeinfo": "cpp",
+        "utility": "cpp",
+        "cstdarg": "cpp",
+        "list": "cpp",
+        "numeric": "cpp",
+        "bitset": "cpp",
+        "cstring": "cpp",
+        "deque": "cpp",
+        "algorithm": "cpp",
+        "codecvt": "cpp"
+    }
+}

week_1/2018202133HPY/Makefile

@@ -0,0 +1,30 @@
+override CXX=clang++
+
+SRC = ${wildcard src/*.cpp}
+OBJ = ${patsubst src/%.cpp, tmp/%.o, $(SRC)}
+DEPEND = ${OBJ:tmp/%.o=tmp/%.d} 
+
+CXXFLAGS = -std=gnu++17 -g -O2
+LDFLAGS = -lcurl -lstdc++fs -lpthread
+
+all:  main
+
+include $(DEPEND) 
+
+#tmp/%.o : src/%.cpp
+#	$(CXX) $(CXXFLAGS) -c $< -o $@
+
+tmp/%.d: src/%.cpp
+	$(CXX) -MM $< | sed 's/^\(\S\)/tmp\/\1/' > $@
+	echo -e '\t$$(CXX) $$(CXXFLAGS) -c $< -o ${patsubst src/%.cpp, tmp/%.o, $<}' >> $@
+
+main: $(OBJ)
+	$(CXX) $(OBJ) -o main $(LDFLAGS)
+
+clean_depend:
+	$(RM) $(DEPEND)
+
+clean:
+	$(RM) depend tmp/*.o main tmp/*.d
+
+.PHONY: clean clean_depend

week_1/2018202133HPY/README.md

@@ -0,0 +1,7 @@
+# Simple crawler 
+---
+## Usage 
+```
+$ make
+$ ./main
+```

week_1/2018202133HPY/src/Curl.cpp

@@ -0,0 +1,16 @@
+#include <curl/curl.h>
+
+class CurlInit {
+    CurlInit()
+    {
+        curl_global_init(CURL_GLOBAL_ALL);
+    }
+    ~CurlInit()
+    {
+        curl_global_cleanup();
+    }
+    static void _()
+    {
+        static CurlInit __curl_init;
+    }
+};

week_1/2018202133HPY/src/CurlEasyHandle.cpp

@@ -0,0 +1,41 @@
+#include "CurlEasyHandle.hpp"
+
+#include <mutex>
+#include <sstream>
+#include <string>
+
+size_t CurlEasyHandle::curlHeaderHandleFunc(void* buffer, size_t size, size_t nmemb, void* userp)
+{
+    auto that = (CurlEasyHandle*)userp;
+
+    std::lock_guard lock_that { that->lock };
+
+    std::string key, val;
+    std::stringstream ss;
+
+    ss.write((const char*)buffer, nmemb);
+
+    if (!(ss >> key)) {
+        that->on_header_receive(*that);
+    } else if (key.substr(0, 4) == "HTTP") {
+        that->env["HTTP_VER"] = key.substr(5);
+
+        ss >> val;
+
+        that->env["STATUS_CODE"] = val;
+
+        getline(ss, val);
+        val.erase(0, val.find_first_not_of(" \r\n\t"));
+        val.erase(val.find_last_not_of(" \r\n\t") + 1);
+        that->env["REASON_PHRASE"] = val;
+    } else {
+        key.erase(key.find_last_of(':'));
+        std::transform(key.begin(), key.end(), key.begin(), ::tolower);
+
+        getline(ss, val);
+        val.erase(0, val.find_first_not_of(" \r\n\t"));
+        val.erase(val.find_last_not_of(" \r\n\t") + 1);
+        that->env[key] = val;
+    }
+    return nmemb;
+}

week_1/2018202133HPY/src/CurlEasyHandle.hpp

@@ -0,0 +1,132 @@
+#ifndef _COWR_CURLEASYHANDLE
+
+#define _COWR_CURLEASYHANDLE
+
+#include <curl/curl.h>
+
+#include <functional>
+#include <iostream>
+#include <map>
+#include <mutex>
+
+class CurlEasyHandle {
+protected:
+    std::recursive_mutex lock;
+    CURL* handle;
+
+    std::map<std::string, std::string> env;
+
+    std::function<void(void*, size_t, size_t, CurlEasyHandle&)> on_data_receive = curlDefaultDataReceiveHandler;
+    std::function<void(CurlEasyHandle&)> on_header_receive = curlDefaultHeaderReceiveHandler;
+
+public:
+    CurlEasyHandle()
+    {
+        handle = curl_easy_init();
+
+        curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, curlDataHandleFunc);
+        curl_easy_setopt(handle, CURLOPT_WRITEDATA, this);
+        curl_easy_setopt(handle, CURLOPT_HEADERFUNCTION, curlHeaderHandleFunc);
+        curl_easy_setopt(handle, CURLOPT_HEADERDATA, this);
+        curl_easy_setopt(handle, CURLOPT_USERAGENT, "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36");
+
+        setCanRedirect(true);
+    }
+
+    ~CurlEasyHandle()
+    {
+        if (handle)
+            curl_easy_cleanup(handle);
+    }
+
+    CurlEasyHandle(const CurlEasyHandle&) = delete;
+
+    CurlEasyHandle(CurlEasyHandle&& r)
+    {
+        std::lock_guard lock_r { r.lock };
+        env = std::move(r.env);
+        on_data_receive = std::move(r.on_data_receive);
+        on_header_receive = std::move(r.on_header_receive);
+        handle = r.handle;
+        r.handle = nullptr;
+    }
+
+    template <typename T>
+    CURLcode setOption(CURLoption option, T args)
+    {
+        std::lock_guard lock_this { lock };
+        return curl_easy_setopt(handle, option, args);
+    }
+
+    CURLcode setURL(std::string URI)
+    {
+        std::lock_guard lock_this { lock };
+        return curl_easy_setopt(handle, CURLOPT_URL, URI.c_str());
+    }
+
+    CURLcode setDataReceiveHandler(std::function<void(void*, size_t, size_t, CurlEasyHandle&)> handler)
+    {
+        std::lock_guard lock_this { lock };
+        on_data_receive = handler;
+        return CURLE_OK;
+    }
+
+    CURLcode setHeaderReceiveHandler(std::function<void(CurlEasyHandle&)> handler)
+    {
+        std::lock_guard lock_this { lock };
+        on_header_receive = handler;
+        return CURLE_OK;
+    }
+
+    CURLcode setCanRedirect(bool redirect)
+    {
+        std::lock_guard lock_this { lock };
+        return curl_easy_setopt(handle, CURLOPT_FOLLOWLOCATION, long(redirect));
+    }
+
+    std::string getURL()
+    {
+        std::lock_guard lock_this { lock };
+        char* url = NULL;
+        curl_easy_getinfo(handle, CURLINFO_EFFECTIVE_URL, &url);
+        return url;
+    }
+
+    virtual CURLcode perform()
+    {
+        std::lock_guard lock_this { lock };
+        return curl_easy_perform(handle);
+    }
+
+    operator CURL*()
+    {
+        return handle;
+    }
+
+    auto getenv() const
+    {
+        std::lock_guard lock_this { const_cast<std::recursive_mutex&>(lock) };
+        return env;
+    }
+
+    static size_t curlDefaultDataReceiveHandler(void* buffer, size_t size, size_t nmemb, CurlEasyHandle& easy_handle)
+    {
+        std::cout.write((const char*)buffer, nmemb);
+        std::cout.flush();
+        return nmemb;
+    }
+
+    static void curlDefaultHeaderReceiveHandler(CurlEasyHandle& easy_handle)
+    {
+    }
+
+    static size_t curlDataHandleFunc(void* buffer, size_t size, size_t nmemb, void* userp)
+    {
+        ((CurlEasyHandle*)userp)->on_data_receive(buffer, size, nmemb, *(CurlEasyHandle*)userp);
+        return nmemb;
+    }
+
+    static size_t curlHeaderHandleFunc(void* buffer, size_t size, size_t nmemb, void* userp);
+};
+
+#endif

week_1/2018202133HPY/src/CurlMultiHandle.cpp

@@ -0,0 +1,61 @@
+#include "CurlMultiHandle.hpp"
+
+int CurlMultiHandle::perform()
+{
+    int running;
+
+    curl_multi_perform(multi_handle, &running);
+
+    struct CURLMsg* m;
+
+    for (auto it = easy_handles.begin(); it != easy_handles.end();) {
+        if ((*it)->can_remove()) {
+            curl_multi_remove_handle(multi_handle, **it);
+
+            lookup.erase(**it);
+            it = easy_handles.erase(it);
+        } else {
+            it++;
+        }
+    }
+
+    if (std::unique_lock lock_this { lock_queue }; easy_handles.size() < max_num && wait_queue.size() > 0) {
+        auto handle = wait_queue.front();
+
+        handle->start();
+        easy_handles.insert(handle);
+        lookup[*handle] = handle;
+        curl_multi_add_handle(multi_handle, *handle);
+
+        wait_queue.pop_front();
+        running = 1;
+    }
+
+    do {
+        int msgq = 0;
+        m = curl_multi_info_read(multi_handle, &msgq);
+        if (m && (m->msg == CURLMSG_DONE)) {
+            CURL* e = m->easy_handle;
+            auto it = easy_handles.find(std::shared_ptr<SingleHandle>(lookup[e]));
+
+            removeHandle(*it);
+        }
+
+    } while (m);
+
+    return running;
+}
+
+int CurlMultiHandle::SingleHandle::curlProgressCallback(void* clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
+{
+    auto psingle = reinterpret_cast<SingleHandle*>(clientp);
+
+    if (!psingle->header_received) {
+        if (getNowTime() - psingle->start_time > (psingle->header_received ? 5 * timeout_ms : timeout_ms)) {
+            psingle->TLE = true;
+            return 1;
+        }
+    }
+
+    return 0;
+}