Skip to content

Commit 70b0af0

Browse files
adazem009adazem009
committed
Add ProjectDownloader class
1 parent 686213e commit 70b0af0

File tree

6 files changed

+492
-6
lines changed

6 files changed

+492
-6
lines changed

‎src/internal/CMakeLists.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ target_sources(scratchcpp
1010
projecturl.h
1111
idownloader.h
1212
idownloaderfactory.h
13+
iprojectdownloader.h
1314
)
1415

1516
if (LIBSCRATCHCPP_NETWORK_SUPPORT)
@@ -19,5 +20,7 @@ if (LIBSCRATCHCPP_NETWORK_SUPPORT)
1920
downloader.h
2021
downloaderfactory.cpp
2122
downloaderfactory.h
23+
projectdownloader.cpp
24+
projectdownloader.h
2225
)
2326
endif()

‎src/internal/iprojectdownloader.h‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
3+
#pragma once
4+
5+
#include <string>
6+
#include <vector>
7+
8+
namespace libscratchcpp
9+
{
10+
11+
class IProjectDownloader
12+
{
13+
public:
14+
virtual ~IProjectDownloader() { }
15+
16+
virtual bool downloadJson(const std::string &projectId) = 0;
17+
virtual bool downloadAssets(const std::vector<std::string> &assetIds) = 0;
18+
virtual void cancel() = 0;
19+
20+
virtual const std::string &json() const = 0;
21+
virtual const std::vector<std::string> &assets() const = 0;
22+
virtual unsigned int downloadedAssetCount() const = 0;
23+
};
24+
25+
} // namespace libscratchcpp

‎src/internal/projectdownloader.cpp‎

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
3+
#include <iostream>
4+
#include <thread>
5+
#include <nlohmann/json.hpp>
6+
7+
#include "projectdownloader.h"
8+
#include "downloaderfactory.h"
9+
#include "idownloader.h"
10+
11+
using namespace libscratchcpp;
12+
13+
static const std::string PROJECT_META_PREFIX = "https://api.scratch.mit.edu/projects/";
14+
static const std::string PROJECT_JSON_PREFIX = "https://projects.scratch.mit.edu/";
15+
static const std::string ASSET_PREFIX = "https://assets.scratch.mit.edu/internalapi/asset/";
16+
static const std::string ASSET_SUFFIX = "/get";
17+
18+
#define CHECK_CANCEL() \
19+
m_cancelMutex.lock(); \
20+
if (m_cancel) { \
21+
m_downloadedAssetCount = 0; \
22+
return false; \
23+
} \
24+
m_cancelMutex.unlock()
25+
26+
ProjectDownloader::ProjectDownloader(IDownloaderFactory *downloaderFactory) :
27+
m_downloaderFactory(downloaderFactory)
28+
{
29+
if (!m_downloaderFactory)
30+
m_downloaderFactory = DownloaderFactory::instance().get();
31+
32+
m_tokenDownloader = m_downloaderFactory->create();
33+
m_jsonDownloader = m_downloaderFactory->create();
34+
}
35+
36+
bool ProjectDownloader::downloadJson(const std::string &projectId)
37+
{
38+
m_cancelMutex.lock();
39+
m_cancel = false;
40+
m_cancelMutex.unlock();
41+
42+
// Get project token
43+
std::cout << "Fetching project info of " << projectId << std::endl;
44+
m_tokenDownloader->startDownload(PROJECT_META_PREFIX + projectId);
45+
m_tokenDownloader->wait();
46+
47+
if (m_tokenDownloader->text().empty()) {
48+
std::cerr << "Could not fetch project info of " << projectId << std::endl;
49+
return false;
50+
}
51+
52+
if (m_tokenDownloader->isCancelled())
53+
return false;
54+
55+
CHECK_CANCEL();
56+
57+
nlohmann::json json;
58+
59+
try {
60+
json = nlohmann::json::parse(m_tokenDownloader->text());
61+
} catch (std::exception &e) {
62+
std::cerr << "Could not parse project info of " << projectId << std::endl;
63+
std::cerr << e.what() << std::endl;
64+
return false;
65+
}
66+
67+
std::string token;
68+
69+
try {
70+
token = json["project_token"];
71+
} catch (std::exception &e) {
72+
std::cerr << "Could not read project token of " << projectId << std::endl;
73+
std::cerr << e.what() << std::endl;
74+
return false;
75+
}
76+
77+
// Download project JSON
78+
std::cout << "Downloading project JSON of " << projectId << std::endl;
79+
m_jsonDownloader->startDownload(PROJECT_JSON_PREFIX + projectId + "?token=" + token);
80+
m_jsonDownloader->wait();
81+
82+
if (m_jsonDownloader->text().empty()) {
83+
std::cerr << "Failed to download project JSON of " << projectId << std::endl;
84+
return false;
85+
}
86+
87+
if (m_jsonDownloader->isCancelled())
88+
return false;
89+
90+
CHECK_CANCEL();
91+
92+
return true;
93+
}
94+
95+
bool ProjectDownloader::downloadAssets(const std::vector<std::string> &assetIds)
96+
{
97+
m_cancelMutex.lock();
98+
m_cancel = false;
99+
m_cancelMutex.unlock();
100+
101+
auto count = assetIds.size();
102+
unsigned int threadCount = std::thread::hardware_concurrency();
103+
unsigned int times; // how many times we should download assets simultaneously (in "groups")
104+
m_assets.clear();
105+
m_downloadedAssetCount = 0;
106+
107+
// Calculate number of "groups"
108+
if (threadCount == 0) {
109+
times = count;
110+
threadCount = 1;
111+
} else
112+
times = std::ceil(count / static_cast<double>(threadCount));
113+
114+
std::cout << "Downloading " << count << " asset(s)";
115+
116+
if (threadCount > 1)
117+
std::cout << " using " << threadCount << " threads";
118+
119+
std::cout << std::endl;
120+
121+
// Create downloaders
122+
std::vector<std::shared_ptr<IDownloader>> downloaders;
123+
124+
for (unsigned int i = 0; i < threadCount; i++)
125+
downloaders.push_back(m_downloaderFactory->create());
126+
127+
// Download assets
128+
for (unsigned int i = 0; i < times; i++) {
129+
unsigned int currentCount = std::min(threadCount, static_cast<unsigned int>(count - i * threadCount));
130+
131+
for (unsigned int j = 0; j < currentCount; j++)
132+
downloaders[j]->startDownload(ASSET_PREFIX + assetIds[i * threadCount + j] + ASSET_SUFFIX);
133+
134+
for (unsigned int j = 0; j < currentCount; j++) {
135+
downloaders[j]->wait();
136+
assert(m_assets.size() == i * threadCount + j);
137+
138+
if (downloaders[j]->isCancelled())
139+
return false;
140+
141+
CHECK_CANCEL();
142+
143+
m_assets.push_back(downloaders[j]->text());
144+
m_downloadedAssetCount++;
145+
}
146+
}
147+
148+
return true;
149+
}
150+
151+
void ProjectDownloader::cancel()
152+
{
153+
m_tokenDownloader->cancel();
154+
m_jsonDownloader->cancel();
155+
m_cancelMutex.lock();
156+
m_cancel = true;
157+
m_cancelMutex.unlock();
158+
m_downloadedAssetCount = 0;
159+
}
160+
161+
const std::string &ProjectDownloader::json() const
162+
{
163+
return m_jsonDownloader->text();
164+
}
165+
166+
const std::vector<std::string> &ProjectDownloader::assets() const
167+
{
168+
return m_assets;
169+
}
170+
171+
unsigned int ProjectDownloader::downloadedAssetCount() const
172+
{
173+
return m_downloadedAssetCount;
174+
}

‎src/internal/projectdownloader.h‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
3+
#pragma once
4+
5+
#include <memory>
6+
#include <atomic>
7+
#include <mutex>
8+
9+
#include "iprojectdownloader.h"
10+
11+
namespace libscratchcpp
12+
{
13+
14+
class IDownloaderFactory;
15+
class IDownloader;
16+
17+
class ProjectDownloader : public IProjectDownloader
18+
{
19+
public:
20+
ProjectDownloader(IDownloaderFactory *downloaderFactory = nullptr);
21+
22+
bool downloadJson(const std::string &projectId) override;
23+
bool downloadAssets(const std::vector<std::string> &assetIds) override;
24+
void cancel() override;
25+
26+
const std::string &json() const override;
27+
const std::vector<std::string> &assets() const override;
28+
unsigned int downloadedAssetCount() const override;
29+
30+
private:
31+
IDownloaderFactory *m_downloaderFactory = nullptr;
32+
std::shared_ptr<IDownloader> m_tokenDownloader;
33+
std::shared_ptr<IDownloader> m_jsonDownloader;
34+
std::vector<std::string> m_assets;
35+
std::atomic<unsigned int> m_downloadedAssetCount = 0;
36+
bool m_cancel = false;
37+
std::mutex m_cancelMutex;
38+
};
39+
40+
} // namespace libscratchcpp

‎test/network/CMakeLists.txt‎

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,16 +15,32 @@ gtest_discover_tests(projecturl_test)
1515
if (LIBSCRATCHCPP_NETWORK_SUPPORT)
1616
# downloaderfactory_test
1717
add_executable(
18-
downloaderfactory_test
19-
downloaderfactory_test.cpp
18+
downloaderfactory_test
19+
downloaderfactory_test.cpp
2020
)
2121

2222
target_link_libraries(
23-
downloaderfactory_test
24-
GTest::gtest_main
25-
scratchcpp
26-
cpr::cpr
23+
downloaderfactory_test
24+
GTest::gtest_main
25+
scratchcpp
26+
cpr::cpr
2727
)
2828

2929
gtest_discover_tests(downloaderfactory_test)
30+
31+
# projectdownloader_test
32+
add_executable(
33+
projectdownloader_test
34+
projectdownloader_test.cpp
35+
)
36+
37+
target_link_libraries(
38+
projectdownloader_test
39+
GTest::gtest_main
40+
GTest::gmock_main
41+
scratchcpp
42+
scratchcpp_mocks
43+
)
44+
45+
gtest_discover_tests(projectdownloader_test)
3046
endif()

0 commit comments

Comments
 (0)