projectsdb/parse_all_projects.go at main · Wildfires-org/projectsdb · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
package main

import (
	"encoding/json"
	"io/ioutil"

	log "github.com/sirupsen/logrus"
)

func ParseAllProjects() error {
	// Get list of forests
	forests, err := GetForests()
	if err != nil {
		return err
	}

	log.WithFields(log.Fields{
		"count": len(forests),
	}).Info("Got forests")

	// For each forest, get the links to the SOPA reports,
	// then parse those pages for project updates
	for i, forest := range forests {
		forests[i] = GetAllForestData(forest)
		if i > 2 {
			break
		}
	}

	return saveProjectsJson(forests)
}

func GetAllForestData(forest Forest) Forest {
	log.WithFields(log.Fields{
		"forest": forest.Name,
		"state":  forest.State,
	}).Info("Looking for projects")

	projectPages, err := GetSopaReportPages(forest.Url)
	if err != nil {
		log.WithFields(log.Fields{
			"forest": forest.Name,
			"state":  forest.State,
			"error":  err.Error(),
		}).Error("Issue getting list of SOPA report urls")
		return forest
	}

	for _, projectPage := range projectPages {
		projects, err := getProjects(projectPage)
		if err != nil {
			log.WithFields(log.Fields{
				"forest": forest.Name,
				"state":  forest.State,
				"page":   projectPage,
				"error":  err.Error(),
			}).Error("Issue getting projects")
		}
		forest.Projects = append(forest.Projects, projects...)
	}

	log.WithFields(log.Fields{
		"forest": forest.Name,
		"state":  forest.State,
		"count":  len(forest.Projects),
	}).Info("Found projects")

	for j := range forest.Projects {
		if len(forest.Projects[j].Id) > 0 {
			log.WithFields(log.Fields{
				"forest":  forest.Name,
				"state":   forest.State,
				"project": forest.Projects[j].Name,
			}).Info("Getting documents")

			docs, err := getReportDocumentMeta(forest.Projects[j].Id)
			if err != nil {
				log.WithFields(log.Fields{
					"forest":  forest.Name,
					"state":   forest.State,
					"project": forest.Projects[j].Name,
				}).Error("Issue getting documents")
				continue
			}

			forest.Projects[j].ProjectDocuments = append(
				forest.Projects[j].ProjectDocuments,
				docs...,
			)

		}
	}

	return forest
}

func saveProjectsJson(forests []Forest) error {
	data, err := json.Marshal(forests)
	if err != nil {
		return err
	}

	return ioutil.WriteFile("data/forests.json", data, 0644)
}