Skip to content

Commit d131cc0

Browse files
tacxoutacxou
committed
Update Makefile and main.py to include 'an' parameter for imports; enhance export_ind to log parameter types
1 parent ca71700 commit d131cc0

File tree

3 files changed

+93
-74
lines changed

3 files changed

+93
-74
lines changed

‎Makefile‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ PLATFORM = "linux/amd64"
88
STC_RUN = "all"
99
STC_IMPORTS = "all"
1010
STC_FORCE = "0"
11+
STC_AN = "0"
1112
include .env
1213

1314
.DEFAULT_GOAL := help
@@ -31,7 +32,7 @@ run-crawler-docker: ## Lance le crawler Sesame - Taiga avec python !
3132
--platform $(PLATFORM) \
3233
--name $(APP_NAME) \
3334
-v $(CURDIR):/data \
34-
$(IMG_NAME) python main.py --run=$(STC_RUN) --imports=$(STC_IMPORTS) --force $(STC_FORCE)
35+
$(IMG_NAME) python main.py --run=$(STC_RUN) --an=$(STC_AN) --imports=$(STC_IMPORTS) --force $(STC_FORCE)
3536

3637
run-crawler: ## Lance le crawler Sesame - Taiga avec python !
3738
@python3 main.py

‎main.py‎

Lines changed: 84 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@
3838
"pass_ensa": hashlib.sha1(ensa_pass.encode()).hexdigest(),
3939
}
4040

41-
# print(ensa_infos)
41+
print(ensa_infos)
4242
# print(headers)
4343

4444
collections = [
@@ -51,75 +51,75 @@
5151
"id": "*",
5252
},
5353
},
54-
# {
55-
# "function": export_ind,
56-
# "method": "ExportInd",
57-
# "params": {
58-
# **ensa_infos,
59-
# "type": "etd",
60-
# "id": "*",
61-
# },
62-
# },
63-
# {
64-
# "function": export_ind,
65-
# "method": "ExportInd",
66-
# "params": {
67-
# **ensa_infos,
68-
# "type": "adm",
69-
# "id": "*",
70-
# },
71-
# },
72-
# {
73-
# "function": export_ind,
74-
# "method": "ExportInd",
75-
# "params": {
76-
# **ensa_infos,
77-
# "type": "esn",
78-
# "id": "*",
79-
# },
80-
# },
81-
# {
82-
# "function": export_pictures,
83-
# "method": "ExportPhotos",
84-
# "methodBase64": "ExportPhoto",
85-
# "params": {
86-
# **ensa_infos,
87-
# "type": "etd",
88-
# "id": "*",
89-
# },
90-
# "paramsBase64": {
91-
# "type": "etd",
92-
# **ensa_infos,
93-
# },
94-
# },
95-
# {
96-
# "function": export_pictures,
97-
# "method": "ExportPhotos",
98-
# "methodBase64": "ExportPhoto",
99-
# "params": {
100-
# **ensa_infos,
101-
# "type": "adm",
102-
# "id": "*",
103-
# },
104-
# "paramsBase64": {
105-
# "type": "adm",
106-
# **ensa_infos,
107-
# },
108-
# },
109-
# {
110-
# "function": export_pictures,
111-
# "method": "ExportPhotos",
112-
# "methodBase64": "ExportPhoto",
113-
# "params": {
114-
# **ensa_infos,
115-
# "type": "esn",
116-
# "id": "*",
117-
# },
118-
# "paramsBase64": {
119-
# "type": "esn",
120-
# **ensa_infos,
121-
# },
122-
# },
54+
{
55+
"function": export_ind,
56+
"method": "ExportInd",
57+
"params": {
58+
**ensa_infos,
59+
"type": "etd",
60+
"id": "*",
61+
},
62+
},
63+
{
64+
"function": export_ind,
65+
"method": "ExportInd",
66+
"params": {
67+
**ensa_infos,
68+
"type": "adm",
69+
"id": "*",
70+
},
71+
},
72+
{
73+
"function": export_ind,
74+
"method": "ExportInd",
75+
"params": {
76+
**ensa_infos,
77+
"type": "esn",
78+
"id": "*",
79+
},
80+
},
81+
{
82+
"function": export_pictures,
83+
"method": "ExportPhotos",
84+
"methodBase64": "ExportPhoto",
85+
"params": {
86+
**ensa_infos,
87+
"type": "etd",
88+
"id": "*",
89+
},
90+
"paramsBase64": {
91+
"type": "etd",
92+
**ensa_infos,
93+
},
94+
},
95+
{
96+
"function": export_pictures,
97+
"method": "ExportPhotos",
98+
"methodBase64": "ExportPhoto",
99+
"params": {
100+
**ensa_infos,
101+
"type": "adm",
102+
"id": "*",
103+
},
104+
"paramsBase64": {
105+
"type": "adm",
106+
**ensa_infos,
107+
},
108+
},
109+
{
110+
"function": export_pictures,
111+
"method": "ExportPhotos",
112+
"methodBase64": "ExportPhoto",
113+
"params": {
114+
**ensa_infos,
115+
"type": "esn",
116+
"id": "*",
117+
},
118+
"paramsBase64": {
119+
"type": "esn",
120+
**ensa_infos,
121+
},
122+
},
123123
]
124124

125125

@@ -130,10 +130,21 @@ async def main():
130130
parser.add_argument('--an', help='Année universitaire à importer',default="0")
131131
parser.add_argument('--force', help="Force l'import et bypass le check du fingerprint",default="0")
132132
args = parser.parse_args()
133-
if args.an != 0:
134-
print(f"Import pour l'annee {args.an}")
133+
if args.an != '0':
134+
print(f"Import pour l'annee <{args.an}>")
135+
if args.an.isdigit():
136+
args.an = int(args.an)
137+
if args.an < 2000 or args.an > datetime.now().year + 1:
138+
print("Année invalide, doit être entre 2000 et l'année en cours + 1")
139+
sys.exit(1)
140+
141+
for col in collections:
142+
col.get('params')['au']=args.an
143+
else:
144+
annee_en_cours = datetime.now().year
145+
print("Import pour l'annee en cours <" + str(annee_en_cours) + ">")
135146
for col in collections:
136-
col.get('params')['au']=int(args.an)
147+
col.get('params')['au']=annee_en_cours
137148

138149
if args.run == 'taiga' or args.run == 'all':
139150
logger.info("Starting Taiga ind/pictures crawler...")

‎src/export_ind.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,13 @@ async def export_ind(url, col, headers):
1818
"params": col.get("params"),
1919
"id": str(uuid.uuid4()),
2020
}
21+
22+
if col.get('params').get('type') == 'pri':
23+
payload['params']['au'] = int(col.get('params').get('au')) + 1
24+
print("Type primo détecté, ajout de +1 à l'année: <" + str(payload['params']['au']) + ">")
25+
else:
26+
print("Type " + col.get('params').get('type') + " détecté, pas d'ajout à l'année: <" + str(col.get('params').get('au')) + ">")
27+
2128
try:
2229
response = requests.post(url, json=payload, headers=headers, verify=False, timeout=10000)
2330
response.raise_for_status()

0 commit comments

Comments
 (0)