Skip to content

Commit c7cd73a

Browse files
MortI2CMortI2C
committed
Added system fragmentation calculation
1 parent 3b71bb1 commit c7cd73a

6 files changed

Lines changed: 224 additions & 6 deletions

File tree

‎bin/scorsa-sched‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,8 +41,9 @@ length = config.getfloat("simulator", "length")
4141
period = config.getfloat("simulator", "period")
4242
digits = config.getint("simulator", "digits")
4343
families = json.loads(config.get("system", "families"))
44+
layout_info = defaultdict(list)
4445

45-
layout = scorsa.map_layout(np.genfromtxt(args.l, delimiter=',', dtype=None))
46+
layout = scorsa.map_layout(np.genfromtxt(args.l, delimiter=',', dtype=None), layout_info)
4647
max_dist = scorsa.distance(layout, 0, len(layout) - 1)
4748

4849
with open(args.w) as w:
@@ -91,14 +92,16 @@ for i in scorsa.steps(0.0, length, period, digits):
9192
# measure fragmentation of free slots, and for each running job,
9293
# normalized by number of sockets
9394
sockets = scorsa.list_free_sockets(free)
94-
f = scorsa.fragmentation(layout, sockets) * len(sockets) / len(layout)
95+
used_sockets = scorsa.list_used_sockets(sockets,layout)
96+
# f = scorsa.fragmentation(layout, sockets) * len(sockets) / len(layout)
97+
f = scorsa.system_fragmentation(layout, used_sockets, layout_info)
9598
r = d = b = 0.0
9699
for jid in running:
97100
sockets = scorsa.list_sockets(schedule[jid]["nodes"])
98101
job_dist = scorsa.distance(layout, min(sockets), max(sockets))
99102
min_dist = scorsa.distance(layout, 0, len(sockets) - 1)
100103
ratio = len(sockets) / len(layout)
101-
f += scorsa.fragmentation(layout, sockets) * len(sockets) / len(layout)
104+
# f += scorsa.fragmentation(layout, sockets) * len(sockets) / len(layout)
102105
d += job_dist - min_dist
103106
r += ratio if schedule[jid]["reused"] else 0.0
104107
b += ratio if schedule[jid]["backscaled"] else 0.0

‎bin/scorsa.py‎

Lines changed: 64 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,20 @@
11
from __future__ import division
22

33
import numpy as np
4+
import math
45

56
from collections import defaultdict
67
from itertools import groupby, chain
78
from operator import itemgetter
89

10+
911
# Convert standard time in float format to discrete time in "step" format.
1012
def step(time, period, digits):
1113
s = time - (time % period)
1214
s = s + period if time % period != 0 else s
1315
return round(s, digits)
1416

17+
1518
# Generate list of time steps between the specified interval. Similar to
1619
# Python's range(), with float support.
1720
def steps(start, stop, period, digits):
@@ -20,7 +23,8 @@ def steps(start, stop, period, digits):
2023
yield round(r, digits)
2124
r += period
2225

23-
def map_layout(data):
26+
27+
def map_layout(data,layout_info):
2428
m = {}
2529
sled_id = 0
2630
draw_id = 0
@@ -50,12 +54,17 @@ def map_layout(data):
5054

5155
sled_id += 1
5256

57+
layout_info["n_racks"] = rack_id+1
58+
layout_info["n_drawers"] = (draw_id+1)*layout_info["n_racks"]
59+
layout_info["sleds_drawer"] = (sled_id / layout_info["n_racks"])/(draw_id+1)
60+
5361
return m
5462

63+
5564
def distance(layout, a, b):
5665
d = 0
5766
if a != b:
58-
d = 1
67+
d = 1
5968
if layout[a]["sled_id"] != layout[b]["sled_id"]:
6069
d = 10
6170
if layout[a]["draw_id"] != layout[b]["draw_id"]:
@@ -64,6 +73,7 @@ def distance(layout, a, b):
6473
d = 1000
6574
return d
6675

76+
6777
def fragmentation(layout, subset):
6878
f = 0.0
6979
by_rack = defaultdict(list)
@@ -83,12 +93,64 @@ def fragmentation(layout, subset):
8393

8494
return f / len(by_rack)
8595

96+
97+
def system_fragmentation(layout, used, layout_info):
98+
blocks_total = math.ceil(layout_info["n_drawers"]/layout_info["n_racks"])
99+
blocks_resources = layout_info["sleds_drawer"]
100+
resources_total = blocks_total * blocks_resources
101+
resources_used = defaultdict(list)
102+
blocks_seen = defaultdict(list)
103+
104+
#Determine which sids are used
105+
for sid in used:
106+
if layout[sid]["rack_id"] not in resources_used:
107+
resources_used[layout[sid]["rack_id"]] = 1
108+
blocks_seen[layout[sid]["rack_id"]] = []
109+
else:
110+
resources_used[layout[sid]["rack_id"]] += 1
111+
112+
if layout[sid]['draw_id'] not in blocks_seen[layout[sid]["rack_id"]]:
113+
blocks_seen[layout[sid]["rack_id"]].append(layout[sid]['draw_id'])
114+
115+
sum_f = 0
116+
for rf in resources_used:
117+
f = 0
118+
blocks_used = len(blocks_seen[rf])
119+
min_blocks = math.ceil((resources_used[rf] / resources_total)*blocks_total)
120+
121+
if min_blocks == 1 and blocks_used == blocks_total:
122+
f = 1
123+
elif min_blocks == 0:
124+
f = 0
125+
elif blocks_used == min_blocks:
126+
f = 0
127+
else:
128+
f = (blocks_used - min_blocks) / blocks_total
129+
130+
sum_f += f
131+
132+
if sum_f == 0:
133+
return 0
134+
else:
135+
return sum_f / layout_info["n_racks"]
136+
137+
86138
def list_sockets(nodes):
87139
return list(chain.from_iterable(nodes))
88140

141+
89142
def list_free_sockets(free):
90143
nodes = []
91144
for f in free.values():
92145
for l, n in f.iteritems():
93146
nodes.append(n)
94147
return list(chain.from_iterable(chain.from_iterable(nodes)))
148+
149+
150+
def list_used_sockets(free,layout):
151+
used = []
152+
for sid in layout:
153+
if sid not in free:
154+
used.append(sid)
155+
156+
return used

‎bin/test-distance‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,15 @@ import numpy as np
1313

1414
import scorsa
1515

16+
from collections import defaultdict
17+
1618
ap = argparse.ArgumentParser()
1719
ap.add_argument("-l", "--layout", dest="l", required=True,
1820
help="System layout file")
1921
args = ap.parse_args()
22+
layout_info = defaultdict(list)
2023

21-
layout = scorsa.map_layout(np.genfromtxt(args.l, delimiter=',', dtype=None))
24+
layout = scorsa.map_layout(np.genfromtxt(args.l, delimiter=',', dtype=None), layout_info)
2225

2326
pids = sorted(layout.keys())
2427
print "-", " ".join(str(pid) for pid in pids)

‎data/frag0125.json‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
[
2+
{
3+
"tasks" : 4,
4+
"time" : 10,
5+
"id" : "0",
6+
"scale" : "up",
7+
"color" : "#0000ff",
8+
"arrival" : 0,
9+
"mem" : 1
10+
},
11+
{
12+
"tasks" : 4,
13+
"time" : 15,
14+
"id" : "1",
15+
"scale" : "up",
16+
"color" : "#ff0000",
17+
"arrival" : 11,
18+
"mem" : 1
19+
},
20+
{
21+
"tasks" : 4,
22+
"time" : 8,
23+
"id" : "2",
24+
"scale" : "up",
25+
"color" : "#00ff00",
26+
"arrival" : 0,
27+
"mem" : 1
28+
},
29+
{
30+
"tasks" : 4,
31+
"time" : 10,
32+
"id" : "3",
33+
"scale" : "up",
34+
"color" : "#ffff00",
35+
"arrival" : 9,
36+
"mem" : 1
37+
}
38+
]

‎data/frag0125_2.json‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
[
2+
{
3+
"tasks" : 8,
4+
"time" : 10,
5+
"id" : "0",
6+
"scale" : "up",
7+
"color" : "#0000ff",
8+
"arrival" : 0,
9+
"mem" : 1
10+
},
11+
{
12+
"tasks" : 4,
13+
"time" : 15,
14+
"id" : "1",
15+
"scale" : "up",
16+
"color" : "#ff0000",
17+
"arrival" : 11,
18+
"mem" : 1
19+
},
20+
{
21+
"tasks" : 4,
22+
"time" : 8,
23+
"id" : "2",
24+
"scale" : "up",
25+
"color" : "#00ff00",
26+
"arrival" : 0,
27+
"mem" : 1
28+
},
29+
{
30+
"tasks" : 4,
31+
"time" : 10,
32+
"id" : "3",
33+
"scale" : "up",
34+
"color" : "#ffff00",
35+
"arrival" : 9,
36+
"mem" : 1
37+
}
38+
]

‎data/frag05.json‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
[
2+
{
3+
"tasks" : 4,
4+
"time" : 10,
5+
"id" : "0",
6+
"scale" : "up",
7+
"color" : "#0000ff",
8+
"arrival" : 0,
9+
"mem" : 1
10+
},
11+
{
12+
"tasks" : 4,
13+
"time" : 20,
14+
"id" : "1",
15+
"scale" : "up",
16+
"color" : "#ff0000",
17+
"arrival" : 0,
18+
"mem" : 1
19+
},
20+
{
21+
"tasks" : 4,
22+
"time" : 10,
23+
"id" : "2",
24+
"scale" : "up",
25+
"color" : "#00ff00",
26+
"arrival" : 0,
27+
"mem" : 1
28+
},
29+
{
30+
"tasks" : 4,
31+
"time" : 20,
32+
"id" : "3",
33+
"scale" : "up",
34+
"color" : "#ff0000",
35+
"arrival" : 0,
36+
"mem" : 1
37+
},
38+
{
39+
"tasks" : 4,
40+
"time" : 10,
41+
"id" : "4",
42+
"scale" : "up",
43+
"color" : "#ee00ee",
44+
"arrival" : 0,
45+
"mem" : 1
46+
},
47+
{
48+
"tasks" : 4,
49+
"time" : 20,
50+
"id" : "5",
51+
"scale" : "up",
52+
"color" : "#cccccc",
53+
"arrival" : 0,
54+
"mem" : 1
55+
},
56+
{
57+
"tasks" : 4,
58+
"time" : 10,
59+
"id" : "6",
60+
"scale" : "up",
61+
"color" : "#00eeee",
62+
"arrival" : 0,
63+
"mem" : 1
64+
},
65+
{
66+
"tasks" : 4,
67+
"time" : 20,
68+
"id" : "7",
69+
"scale" : "up",
70+
"color" : "#0000ee",
71+
"arrival" : 0,
72+
"mem" : 1
73+
}
74+
]

0 commit comments

Comments
 (0)