@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
@prefix hobbit: <http://w3id.org/hobbit/vocab#> .
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
@prefix owl: <http://www.w3.org/2002/07/owl#> .
@prefix bench: <http://w3id.org/bench#> .
@prefix qb: <http://purl.org/linked-data/cube#> .
#-------------------------------------------------------------------------------------------------------------------------------
bench:OdinBenchmark a hobbit:Benchmark;
rdfs:label "ODIN Benchmark"@en;
rdfs:comment "HOBBIT T3.1 benchmark ODIN"@en;
hobbit:hasAPI bench:OdinApi;
hobbit:imageName "git.project-hobbit.eu:4567/kleanthie.georgala/odinbenchmarkcontroller";
hobbit:measuresKPI bench:averageTaskDelay ;
hobbit:measuresKPI bench:microAverageRecall ;
hobbit:measuresKPI bench:microAveragePrecision ;
hobbit:measuresKPI bench:microAverageFmeasure ;
hobbit:measuresKPI bench:macroAverageRecall ;
hobbit:measuresKPI bench:macroAveragePrecision ;
hobbit:measuresKPI bench:macroAverageFmeasure ;
hobbit:measuresKPI bench:maxTPS ;
hobbit:measuresKPI bench:averageTPS ;
hobbit:measuresKPI bench:tasksRecall ;
hobbit:measuresKPI bench:tasksPrecision ;
hobbit:measuresKPI bench:tasksFmeasure ;
hobbit:measuresKPI bench:tasksTPS ;
hobbit:measuresKPI bench:tasksAnswerDelay ;
hobbit:hasParameter bench:hasNumberOfDataGenerators;
hobbit:hasParameter bench:hasNumberOfTaskGenerators;
hobbit:hasParameter bench:hasSeed;
hobbit:hasParameter bench:hasPopulation;
hobbit:hasParameter bench:hasMimickingAlgorithm;
hobbit:hasParameter bench:hasMimickingOutput;
hobbit:hasParameter bench:hasNumberOfInsertQueries;
hobbit:hasParameter bench:hasDuration.
bench:OdinApi a hobbit:API .
# --- Parameters ---
bench:hasNumberOfDataGenerators a hobbit:Parameter, hobbit:FeatureParameter, hobbit:ConfigurableParameter;
rdfs:label "Number of data generators - agents"@en;
rdfs:comment "The number of independent data generators that will insert data into the triple stores."@en;
rdfs:domain hobbit:Experiment, hobbit:Challenge;
rdfs:range xsd:unsignedInt;
hobbit:defaultValue "10"^^xsd:unsignedInt .
bench:hasNumberOfTaskGenerators a hobbit:Parameter, hobbit:FeatureParameter, hobbit:ConfigurableParameter;
rdfs:label "Number of task generators - agents"@en;
rdfs:comment "The number of independent task generators that perform select queries against the triple stores."@en;
rdfs:domain hobbit:Experiment, hobbit:Challenge;
rdfs:range xsd:unsignedInt;
hobbit:defaultValue "1"^^xsd:unsignedInt .
bench:hasSeed a hobbit:Parameter, hobbit:FeatureParameter, hobbit:ConfigurableParameter;
rdfs:label "Seed for mimicking algorithm"@en;
rdfs:comment "The seed used by the mimicking algorithm."@en;
rdfs:domain hobbit:Experiment, hobbit:Challenge;
rdfs:range xsd:unsignedInt;
hobbit:defaultValue "100"^^xsd:unsignedInt .
bench:hasPopulation a hobbit:Parameter, hobbit:FeatureParameter, hobbit:ConfigurableParameter;
rdfs:label "Population of generated data"@en;
rdfs:comment "Number of different events to be generated by a mimicking algorithm"@en;
rdfs:domain hobbit:Experiment, hobbit:Challenge;
rdfs:range xsd:unsignedInt;
hobbit:defaultValue "1000"^^xsd:unsignedInt .
bench:hasMimickingAlgorithm a hobbit:Parameter, hobbit:FeatureParameter, hobbit:ConfigurableParameter;
rdfs:label "Name of mimicking algorithm"@en;
rdfs:comment "The mimicking algorithm used by the benchmark to generate data for ingestion."@en;
rdfs:domain hobbit:Experiment, hobbit:Challenge;
rdfs:range xsd:string;
hobbit:defaultValue "MIMICKING_SIMPLE"^^xsd:string.
bench:hasMimickingOutput a hobbit:Parameter, hobbit:FeatureParameter, hobbit:ConfigurableParameter;
rdfs:label "Name of mimicking algorithm output folder"@en;
rdfs:comment "The mimicking algorithm output folder used to store the data generated from a mimicing algorithm."@en;
rdfs:domain hobbit:Experiment, hobbit:Challenge;
rdfs:range xsd:string.
bench:hasNumberOfInsertQueries a hobbit:Parameter, hobbit:FeatureParameter, hobbit:ConfigurableParameter;
rdfs:label "Number of insert queries"@en;
rdfs:comment "Number of insert queries performed by a data generator until a select query is performed."@en;
rdfs:domain hobbit:Experiment, hobbit:Challenge;
rdfs:range xsd:unsignedInt;
hobbit:defaultValue "10"^^xsd:unsignedInt .
bench:hasDuration a hobbit:Parameter, hobbit:FeatureParameter, hobbit:ConfigurableParameter;
rdfs:label "Duration of the benchmark"@en;
rdfs:comment "Duration of the benchmark"@en;
rdfs:domain hobbit:Experiment, hobbit:Challenge;
rdfs:range xsd:unsignedLong;
hobbit:defaultValue "10800000"^^xsd:unsignedLong .
# --- KPIs ---
bench:tasksRecall a hobbit:KPI ;
rdfs:label "Evaluation of each SELECT SPARQL query"@en;
rdfs:comment "Evaluation of Recall"@en;
rdfs:range qb:DataSet .
bench:tasksPrecision a hobbit:KPI ;
rdfs:label "Evaluation of each SELECT SPARQL query"@en;
rdfs:comment "Evaluation of Precision"@en;
rdfs:range qb:DataSet .
bench:tasksFmeasure a hobbit:KPI ;
rdfs:label "Evaluation of each SELECT SPARQL query"@en;
rdfs:comment "Evaluation of Fmeasure"@en;
rdfs:range qb:DataSet .
bench:tasksTPS a hobbit:KPI ;
rdfs:label "Evaluation of each SELECT SPARQL query"@en;
rdfs:comment "Evaluation of Triples-Per-Seconds"@en;
rdfs:range qb:DataSet .
bench:tasksAnswerDelay a hobbit:KPI ;
rdfs:label "Evaluation of each SELECT SPARQL query"@en;
rdfs:comment "Evaluation of Answer Delay"@en;
rdfs:range qb:DataSet .
bench:maxTPS a hobbit:KPI;
rdfs:label "Maximum Triples-Per-Seconds"@en;
rdfs:comment "Max TPS achieved by the system"@en;
rdfs:range xsd:double .
bench:averageTPS a hobbit:KPI;
rdfs:label "Average Triples-Per-Seconds"@en;
rdfs:comment "Average TPS achieved by the system"@en;
rdfs:range xsd:double .
# --- Micro Averaging ---
bench:averageTaskDelay a hobbit:KPI;
rdfs:label "Average Delay of Tasks"@en;
rdfs:comment "Average time need to complete a select query"@en;
rdfs:range xsd:double .
bench:microAverageRecall a hobbit:KPI;
rdfs:label "Micro-Average-Recall"@en;
rdfs:comment "Micro-Average-Recall = (sum of TP of all tasks) / (sum of TP of all tasks + sum of FN of all tasks)"@en;
rdfs:range xsd:double .
bench:microAveragePrecision a hobbit:KPI;
rdfs:label "Micro-Average-Precision"@en;
rdfs:comment "Micro-Average-Precision = (sum of TP of all tasks) / (sum of TP of all tasks + sum of FP of all tasks)"@en;
rdfs:range xsd:double .
bench:microAverageFmeasure a hobbit:KPI;
rdfs:label "Micro-Average-F-Measure"@en;
rdfs:comment "Micro-Average-F-Measure = (2 * Micro-Average-Precision * Micro-Average-Recall) / (Micro-Average-Precision + Micro-Average-Recall)"@en;
rdfs:range xsd:double .
# --- Macro Averaging ---
bench:macroAverageRecall a hobbit:KPI;
rdfs:label "Macro-Average-Recall"@en;
rdfs:comment "Macro-Average-Recall = (sum of Recall of all tasks) / (number of all tasks)"@en;
rdfs:range xsd:double .
bench:macroAveragePrecision a hobbit:KPI;
rdfs:label "Macro-Average-Precision"@en;
rdfs:comment "Macro-Average-Precision = (sum of Precision of all tasks) / (number of all tasks)"@en;
rdfs:range xsd:double .
bench:macroAverageFmeasure a hobbit:KPI;
rdfs:label "Macro-Average-F-Measure"@en;
rdfs:comment "Macro-Average-F-Measure = (2 * Macro-Average-Recall * Macro-Average-Precision) / (Macro-Average-Precision + Macro-Average-Recall)"@en;
rdfs:range xsd:double .