1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
@prefix hobbit: <http://w3id.org/hobbit/vocab#> .
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
@prefix owl: <http://www.w3.org/2002/07/owl#> .
@prefix bench: <http://w3id.org/bench#> .
@prefix qb: <http://purl.org/linked-data/cube#> .
@prefix mocha: <https://project-hobbit.eu/challenges/>
#-------------------------------------------------------------------------------------------------------------------------------
bench:OdinBenchmark a hobbit:Benchmark;
rdfs:label "ODIN Benchmark Version 1"@en;
rdfs:comment "HOBBIT benchmark ODIN"@en;
hobbit:hasAPI mocha:MOCHA2017-API;
hobbit:imageName "git.project-hobbit.eu:4567/kleanthie.georgala/odinbenchmarkcontroller:1.0.0";
hobbit:usesImage "git.project-hobbit.eu:4567/kleanthie.georgala/odindatagenerator:1.0.0";
hobbit:usesImage "git.project-hobbit.eu:4567/kleanthie.georgala/odintaskgenerator:1.0.0";
hobbit:usesImage "git.project-hobbit.eu:4567/kleanthie.georgala/odinevaluationmodule:1.0.0";
hobbit:version "1.0.0";
hobbit:measuresKPI bench:averageTaskDelay ;
hobbit:measuresKPI bench:microAverageRecall ;
hobbit:measuresKPI bench:microAveragePrecision ;
hobbit:measuresKPI bench:microAverageFmeasure ;
hobbit:measuresKPI bench:macroAverageRecall ;
hobbit:measuresKPI bench:macroAveragePrecision ;
hobbit:measuresKPI bench:macroAverageFmeasure ;
hobbit:measuresKPI bench:maxTPS ;
hobbit:measuresKPI bench:averageTPS ;
hobbit:measuresKPI bench:tasksRecall ;
hobbit:measuresKPI bench:tasksPrecision ;
hobbit:measuresKPI bench:tasksFmeasure ;
hobbit:measuresKPI bench:tasksTPS ;
hobbit:measuresKPI bench:tasksAnswerDelay ;
hobbit:hasParameter bench:hasNumberOfDataGenerators;
hobbit:hasParameter bench:hasNumberOfTaskGenerators;
hobbit:hasParameter bench:hasSeed;
hobbit:hasParameter bench:hasPopulation;
hobbit:hasParameter bench:hasMimickingAlgorithm;
hobbit:hasParameter bench:hasMimickingOutput;
hobbit:hasParameter bench:hasNumberOfInsertQueries;
hobbit:hasParameter bench:hasDuration.
mocha:MOCHA2017-API a hobbit:API .
# --- Parameters ---
bench:hasNumberOfDataGenerators a hobbit:Parameter, hobbit:FeatureParameter, hobbit:ConfigurableParameter;
rdfs:label "Number of data generators - agents"@en;
rdfs:comment "The number of independent data generators that will insert data into the triple stores."@en;
rdfs:domain hobbit:Experiment, hobbit:Challenge;
rdfs:range xsd:unsignedInt;
hobbit:defaultValue "2"^^xsd:unsignedInt .
bench:hasNumberOfTaskGenerators a hobbit:Parameter, hobbit:FeatureParameter, hobbit:ConfigurableParameter;
rdfs:label "Number of task generators - agents"@en;
rdfs:comment "The number of independent task generators that perform select queries against the triple stores."@en;
rdfs:domain hobbit:Experiment, hobbit:Challenge;
rdfs:range xsd:unsignedInt;
hobbit:defaultValue "1"^^xsd:unsignedInt .
bench:hasSeed a hobbit:Parameter, hobbit:FeatureParameter, hobbit:ConfigurableParameter;
rdfs:label "Seed for mimicking algorithm"@en;
rdfs:comment "The seed used by the mimicking algorithm."@en;
rdfs:domain hobbit:Experiment, hobbit:Challenge;
rdfs:range xsd:unsignedInt;
hobbit:defaultValue "100"^^xsd:unsignedInt .
bench:hasPopulation a hobbit:Parameter, hobbit:FeatureParameter, hobbit:ConfigurableParameter;
rdfs:label "Population of generated data"@en;
rdfs:comment "Number of different events to be generated by a mimicking algorithm"@en;
rdfs:domain hobbit:Experiment, hobbit:Challenge;
rdfs:range xsd:unsignedInt;
hobbit:defaultValue "1000"^^xsd:unsignedInt .
bench:hasMimickingAlgorithm a hobbit:Parameter, hobbit:FeatureParameter, hobbit:ConfigurableParameter;
rdfs:label "Name of mimicking algorithm"@en;
rdfs:comment "The mimicking algorithm used by the benchmark to generate data for ingestion."@en;
rdfs:domain hobbit:Experiment, hobbit:Challenge;
rdfs:range xsd:string;
hobbit:defaultValue "TRANSPORT_DATA"^^xsd:string.
bench:hasMimickingOutput a hobbit:Parameter, hobbit:FeatureParameter, hobbit:ConfigurableParameter;
rdfs:label "Name of mimicking algorithm output folder"@en;
rdfs:comment "The mimicking algorithm output folder used to store the data generated from a mimicing algorithm."@en;
rdfs:domain hobbit:Experiment, hobbit:Challenge;
rdfs:range xsd:string;
hobbit:defaultValue "output_data/"^^xsd:string.
bench:hasNumberOfInsertQueries a hobbit:Parameter, hobbit:FeatureParameter, hobbit:ConfigurableParameter;
rdfs:label "Number of insert queries per stream"@en;
rdfs:comment "Number of insert queries performed by a data generator until a select query is performed."@en;
rdfs:domain hobbit:Experiment, hobbit:Challenge;
rdfs:range xsd:unsignedInt;
hobbit:defaultValue "100"^^xsd:unsignedInt .
bench:hasDuration a hobbit:Parameter, hobbit:FeatureParameter, hobbit:ConfigurableParameter;
rdfs:label "Duration of the benchmark"@en;
rdfs:comment "Duration of the benchmark"@en;
rdfs:domain hobbit:Experiment, hobbit:Challenge;
rdfs:range xsd:unsignedLong;
hobbit:defaultValue "600000"^^xsd:unsignedLong .
# --- KPIs ---
bench:tasksRecall a hobbit:KPI ;
rdfs:label "Evaluation of Recall for each SELECT SPARQL query"@en;
rdfs:comment "Evaluation of Recall"@en;
rdfs:range qb:DataSet ;
hobbit:ranking hobbit:DescendingOrder .
bench:tasksPrecision a hobbit:KPI ;
rdfs:label "Evaluation of Precision for each SELECT SPARQL query"@en;
rdfs:comment "Evaluation of Precision"@en;
rdfs:range qb:DataSet ;
hobbit:ranking hobbit:DescendingOrder .
bench:tasksFmeasure a hobbit:KPI ;
rdfs:label "Evaluation of F-measure for each SELECT SPARQL query"@en;
rdfs:comment "Evaluation of Fmeasure"@en;
rdfs:range qb:DataSet ;
hobbit:ranking hobbit:DescendingOrder .
bench:tasksTPS a hobbit:KPI ;
rdfs:label "Evaluation of Triple-Per-Second for each SELECT SPARQL query"@en;
rdfs:comment "Evaluation of Triples-Per-Second"@en;
rdfs:range qb:DataSet ;
hobbit:ranking hobbit:DescendingOrder .
bench:tasksAnswerDelay a hobbit:KPI ;
rdfs:label "Evaluation of task delay for each SELECT SPARQL query"@en;
rdfs:comment "Evaluation of Answer Delay"@en;
rdfs:range qb:DataSet ;
hobbit:ranking hobbit:AscendingOrder .
bench:maxTPS a hobbit:KPI;
rdfs:label "Maximum Triples-Per-Seconds"@en;
rdfs:comment "Max TPS achieved by the system"@en;
rdfs:range xsd:double ;
hobbit:ranking hobbit:DescendingOrder .
bench:averageTPS a hobbit:KPI;
rdfs:label "Average Triples-Per-Seconds"@en;
rdfs:comment "Average TPS achieved by the system"@en;
rdfs:range xsd:double ;
hobbit:ranking hobbit:DescendingOrder .
# --- Micro Averaging ---
bench:averageTaskDelay a hobbit:KPI;
rdfs:label "Average Delay of Tasks"@en;
rdfs:comment "Average time need to complete a select query in seconds"@en;
rdfs:range xsd:double ;
hobbit:ranking hobbit:AscendingOrder .
bench:microAverageRecall a hobbit:KPI;
rdfs:label "Micro-Average-Recall"@en;
rdfs:comment "Micro-Average-Recall = (sum of TP of all tasks) / (sum of TP of all tasks + sum of FN of all tasks)"@en;
rdfs:range xsd:double ;
hobbit:ranking hobbit:DescendingOrder .
bench:microAveragePrecision a hobbit:KPI;
rdfs:label "Micro-Average-Precision"@en;
rdfs:comment "Micro-Average-Precision = (sum of TP of all tasks) / (sum of TP of all tasks + sum of FP of all tasks)"@en;
rdfs:range xsd:double ;
hobbit:ranking hobbit:DescendingOrder .
bench:microAverageFmeasure a hobbit:KPI;
rdfs:label "Micro-Average-F-Measure"@en;
rdfs:comment "Micro-Average-F-Measure = (2 * Micro-Average-Precision * Micro-Average-Recall) / (Micro-Average-Precision + Micro-Average-Recall)"@en;
rdfs:range xsd:double ;
hobbit:ranking hobbit:DescendingOrder .
# --- Macro Averaging ---
bench:macroAverageRecall a hobbit:KPI;
rdfs:label "Macro-Average-Recall"@en;
rdfs:comment "Macro-Average-Recall = (sum of Recall of all tasks) / (number of all tasks)"@en;
rdfs:range xsd:double ;
hobbit:ranking hobbit:DescendingOrder .
bench:macroAveragePrecision a hobbit:KPI;
rdfs:label "Macro-Average-Precision"@en;
rdfs:comment "Macro-Average-Precision = (sum of Precision of all tasks) / (number of all tasks)"@en;
rdfs:range xsd:double ;
hobbit:ranking hobbit:DescendingOrder .
bench:macroAverageFmeasure a hobbit:KPI;
rdfs:label "Macro-Average-F-Measure"@en;
rdfs:comment "Macro-Average-F-Measure = (2 * Macro-Average-Recall * Macro-Average-Precision) / (Macro-Average-Precision + Macro-Average-Recall)"@en;
rdfs:range xsd:double ;
hobbit:ranking hobbit:DescendingOrder .