Skip to content
This repository was archived by the owner on Jan 31, 2026. It is now read-only.

Commit 5cef941

Browse files
EduardoOliveiraEduardoOliveira
authored
Merge pull request #133 from Maker-Management-Platform/discovery-rework
Rework
2 parents 6ffc035 + d3c24c0 commit 5cef941

30 files changed

Lines changed: 708 additions & 443 deletions

‎.dockerignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
testdata*

‎core/api/projects/assets/new.go‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,8 @@ import (
1010
"github.com/eduardooliveira/stLib/core/data/database"
1111
"github.com/eduardooliveira/stLib/core/downloader/tools"
1212
"github.com/eduardooliveira/stLib/core/entities"
13-
"github.com/eduardooliveira/stLib/core/processing"
13+
"github.com/eduardooliveira/stLib/core/processing/initialization"
14+
"github.com/eduardooliveira/stLib/core/processing/types"
1415
"github.com/eduardooliveira/stLib/core/utils"
1516
"github.com/labstack/echo/v4"
1617
"gorm.io/gorm"
@@ -56,13 +57,19 @@ func New(c echo.Context) error {
5657
defer src.Close()
5758
if err = tools.SaveFile(filepath.Join(path, files[0].Filename), src); err != nil {
5859
log.Println(err)
59-
return c.NoContent(http.StatusInternalServerError)
60+
return echo.NewHTTPError(http.StatusInternalServerError, err.Error())
6061
}
61-
processing.EnqueueInitJob(&processing.ProcessableAsset{
62+
63+
_, err = initialization.NewAssetIniter(&types.ProcessableAsset{
6264
Name: files[0].Filename,
6365
Project: project,
6466
Origin: "fs",
65-
})
67+
}).Init()
68+
69+
if err != nil {
70+
log.Println(err)
71+
return echo.NewHTTPError(http.StatusInternalServerError, err.Error())
72+
}
6673

6774
return c.NoContent(http.StatusCreated)
6875
}

‎core/api/projects/discoverHandler.go‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,9 @@ import (
66
"net/http"
77

88
"github.com/eduardooliveira/stLib/core/data/database"
9-
"github.com/eduardooliveira/stLib/core/processing"
9+
"github.com/eduardooliveira/stLib/core/processing/discovery"
10+
"github.com/eduardooliveira/stLib/core/processing/initialization"
11+
"github.com/eduardooliveira/stLib/core/processing/types"
1012
"github.com/labstack/echo/v4"
1113
"gorm.io/gorm"
1214
)
@@ -28,9 +30,14 @@ func discoverHandler(c echo.Context) error {
2830
return echo.NewHTTPError(http.StatusInternalServerError, err.Error())
2931
}
3032

31-
_, err = processing.HandlePath(project.FullPath())
33+
_, err = initialization.NewProjectIniter(types.ProcessableProject{
34+
Path: project.FullPath(),
35+
}).
36+
WithAssetDiscoverer(discovery.FlatAssetDiscoverer{}).
37+
Init()
38+
3239
if err != nil {
33-
log.Printf("error discovering the project %q: %v\n", project.FullPath(), err)
40+
log.Println(err)
3441
return echo.NewHTTPError(http.StatusInternalServerError, err.Error())
3542
}
3643

‎core/api/projects/new.go‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,9 @@ import (
1111

1212
"github.com/eduardooliveira/stLib/core/data/database"
1313
"github.com/eduardooliveira/stLib/core/entities"
14-
"github.com/eduardooliveira/stLib/core/processing"
14+
"github.com/eduardooliveira/stLib/core/processing/discovery"
15+
"github.com/eduardooliveira/stLib/core/processing/initialization"
16+
"github.com/eduardooliveira/stLib/core/processing/types"
1517
"github.com/eduardooliveira/stLib/core/utils"
1618
"github.com/labstack/echo/v4"
1719
)
@@ -83,11 +85,16 @@ func new(c echo.Context) error {
8385
}
8486

8587
}
86-
var project *entities.Project
87-
if project, err = processing.HandlePath(projectFolder); err != nil {
88-
log.Printf("error loading the project %q: %v\n", path, err)
88+
89+
pp, err := initialization.NewProjectIniter(types.ProcessableProject{
90+
Path: projectFolder,
91+
}).WithAssetDiscoverer(discovery.FlatAssetDiscoverer{}).
92+
Init()
93+
if err != nil {
94+
log.Println(err)
8995
return echo.NewHTTPError(http.StatusInternalServerError, err.Error())
9096
}
97+
project := pp.Project
9198

9299
project.Description = createProject.Description
93100
project.Tags = createProject.Tags

‎core/api/system/endpoints.go‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package system
22

33
import (
4+
"context"
45
"errors"
56
"fmt"
67
"io/fs"
@@ -67,7 +68,12 @@ func saveSettings(c echo.Context) error {
6768
}
6869

6970
func runDiscovery(c echo.Context) error {
70-
go processing.Run(runtime.Cfg.Library.Path)
71+
go func() {
72+
err := processing.ProcessFolder(context.Background(), runtime.Cfg.Library.Path)
73+
if err != nil {
74+
fmt.Println(err)
75+
}
76+
}()
7177
return c.NoContent(http.StatusOK)
7278
}
7379

‎core/api/tempfiles/endpoints.go‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,8 @@ import (
99
"github.com/duke-git/lancet/v2/maputil"
1010
"github.com/eduardooliveira/stLib/core/data/database"
1111
models "github.com/eduardooliveira/stLib/core/entities"
12-
"github.com/eduardooliveira/stLib/core/processing"
12+
"github.com/eduardooliveira/stLib/core/processing/initialization"
13+
"github.com/eduardooliveira/stLib/core/processing/types"
1314
"github.com/eduardooliveira/stLib/core/runtime"
1415
"github.com/eduardooliveira/stLib/core/state"
1516
"github.com/eduardooliveira/stLib/core/utils"
@@ -60,11 +61,16 @@ func move(c echo.Context) error {
6061
return echo.NewHTTPError(http.StatusInternalServerError, err.Error())
6162
}
6263

63-
processing.EnqueueInitJob(&processing.ProcessableAsset{
64+
_, err = initialization.NewAssetIniter(&types.ProcessableAsset{
6465
Name: tempFile.Name,
6566
Project: project,
6667
Origin: "fs",
67-
})
68+
}).Init()
69+
70+
if err != nil {
71+
log.Println(err)
72+
return echo.NewHTTPError(http.StatusInternalServerError, err.Error())
73+
}
6874

6975
delete(state.TempFiles, uuid)
7076
return c.NoContent(http.StatusOK)

‎core/downloader/makerworld/makerworld.go‎

Lines changed: 41 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import (
1919
"github.com/eduardooliveira/stLib/core/data/database"
2020
"github.com/eduardooliveira/stLib/core/downloader/tools"
2121
"github.com/eduardooliveira/stLib/core/entities"
22+
"github.com/eduardooliveira/stLib/core/processing/types"
2223
"github.com/eduardooliveira/stLib/core/utils"
2324
"golang.org/x/net/html"
2425
)
@@ -73,12 +74,19 @@ func Fetch(urlString string, cookies []*http.Cookie, userAgent string) error {
7374
return err
7475
}
7576

76-
assets := make([]*entities.ProjectAsset, 0)
77+
assets := make([]*types.ProcessableAsset, 0)
7778
as, err := mwc.fetchCover()
7879
if err != nil {
7980
log.Println("error fetching cover")
8081
return err
8182
}
83+
84+
for _, a := range as {
85+
if a.Asset != nil {
86+
project.DefaultImageID = a.Asset.ID
87+
}
88+
}
89+
8290
assets = append(assets, as...)
8391

8492
as, err = mwc.fetchModels()
@@ -88,75 +96,83 @@ func Fetch(urlString string, cookies []*http.Cookie, userAgent string) error {
8896
}
8997
assets = append(assets, as...)
9098

91-
err = mwc.fetchInstances()
99+
_, err = mwc.fetchInstances()
92100
if err != nil {
93101
log.Println("error fetching models")
94102
return err
95103
}
96104

97-
err = mwc.fetchPictures()
105+
as, err = mwc.fetchPictures()
98106
if err != nil {
99107
log.Println("error fetching pictures")
100108
return err
101109
}
110+
assets = append(assets, as...)
111+
112+
if project.DefaultImageID == "" {
113+
for _, a := range assets {
114+
if a.Asset != nil && a.Asset.AssetType == "image" {
115+
project.DefaultImageID = a.Asset.ID
116+
break
117+
}
118+
}
119+
}
102120

103121
return database.InsertProject(project)
104122
}
105123

106-
func (mwc *mwClient) fetchCover() (assets []*entities.ProjectAsset, err error) {
124+
func (mwc *mwClient) fetchCover() ([]*types.ProcessableAsset, error) {
107125
req, err := http.NewRequest("GET", mwc.metadata.Props.PageProps.Design.CoverURL, nil)
108126
if err != nil {
109127
return nil, err
110128
}
111129
req.Header.Add("User-Agent", mwc.userAgent)
112130

113-
assets = make([]*entities.ProjectAsset, 0)
114-
err = tools.DownloadAsset(path.Base(mwc.metadata.Props.PageProps.Design.CoverURL), mwc.project, mwc.client, req)
115-
if err != nil {
116-
log.Println("Error fetchig cover, skiping: ", err)
117-
return
118-
}
119-
120-
return
131+
return tools.DownloadAsset(path.Base(mwc.metadata.Props.PageProps.Design.CoverURL), mwc.project, mwc.client, req)
121132
}
122133

123-
func (mwc *mwClient) fetchPictures() (err error) {
134+
func (mwc *mwClient) fetchPictures() ([]*types.ProcessableAsset, error) {
135+
assets := make([]*types.ProcessableAsset, 0)
124136
for _, p := range mwc.metadata.Props.PageProps.Design.DesignExtension.DesignPictures {
125137
req, err := http.NewRequest("GET", p.URL, nil)
126138
if err != nil {
127-
return err
139+
return nil, err
128140
}
129141
req.Header.Add("User-Agent", mwc.userAgent)
130142

131-
err = tools.DownloadAsset(p.Name, mwc.project, mwc.client, req)
143+
a, err := tools.DownloadAsset(p.Name, mwc.project, mwc.client, req)
132144
if err != nil {
133145
log.Println("Error fetchig image, skiping: ", err)
134146
continue
135147
}
148+
assets = append(assets, a...)
136149
}
137150

138-
return
151+
return assets, nil
139152
}
140153

141-
func (mwc *mwClient) fetchModels() (assets []*entities.ProjectAsset, err error) {
154+
func (mwc *mwClient) fetchModels() ([]*types.ProcessableAsset, error) {
155+
assets := make([]*types.ProcessableAsset, 0)
142156
for _, m := range mwc.metadata.Props.PageProps.Design.DesignExtension.ModelFiles {
143157
req, err := http.NewRequest("GET", m.ModelURL, nil)
144158
if err != nil {
145159
return nil, err
146160
}
147161
req.Header.Add("User-Agent", mwc.userAgent)
148162

149-
err = tools.DownloadAsset(m.ModelName, mwc.project, mwc.client, req)
163+
a, err := tools.DownloadAsset(m.ModelName, mwc.project, mwc.client, req)
150164
if err != nil {
151165
log.Println("Error fetchig model, skiping: ", err)
152166
continue
153167
}
168+
assets = append(assets, a...)
154169
}
155170

156-
return
171+
return assets, nil
157172
}
158173

159-
func (mwc *mwClient) fetchInstances() (err error) {
174+
func (mwc *mwClient) fetchInstances() ([]*types.ProcessableAsset, error) {
175+
assets := make([]*types.ProcessableAsset, 0)
160176

161177
for _, m := range mwc.metadata.Props.PageProps.Design.Instances {
162178
sl := rand.Intn(6000-3000) + 3000
@@ -169,7 +185,7 @@ func (mwc *mwClient) fetchInstances() (err error) {
169185
}
170186
req, err := http.NewRequest("GET", mfData.URL, nil)
171187
if err != nil {
172-
return err
188+
return nil, err
173189
}
174190
req.Header.Add("User-Agent", mwc.userAgent)
175191

@@ -180,14 +196,16 @@ func (mwc *mwClient) fetchInstances() (err error) {
180196
sl = rand.Intn(6000-3000) + 3000
181197
log.Println("sleeping: ", sl)
182198
time.Sleep(time.Duration(sl) * time.Millisecond)
183-
err = tools.DownloadAsset(name, mwc.project, mwc.client, req)
199+
200+
a, err := tools.DownloadAsset(name, mwc.project, mwc.client, req)
184201
if err != nil {
185202
log.Println("Failed to download 3MF File, skipping")
186203
continue
187204
}
205+
assets = append(assets, a...)
188206
}
189207

190-
return
208+
return assets, nil
191209
}
192210

193211
func (mwc *mwClient) fetch3MFData(id int) (*mf, error) {

0 commit comments

Comments
 (0)