@prefix rdf: . @prefix rdfs: . @prefix hobbit: . @prefix xsd: . @prefix owl: . @prefix bench: . @prefix qb: . @prefix mocha: #------------------------------------------------------------------------------------------------------------------------------- bench:OdinBenchmark a hobbit:Benchmark; rdfs:label "ODIN Benchmark Version 1"@en; rdfs:comment "HOBBIT benchmark ODIN"@en; hobbit:hasAPI mocha:MOCHA2017-API; hobbit:imageName "git.project-hobbit.eu:4567/kleanthie.georgala/odinbenchmarkcontroller:1.0.0"; hobbit:usesImage "git.project-hobbit.eu:4567/kleanthie.georgala/odindatagenerator:1.0.0"; hobbit:usesImage "git.project-hobbit.eu:4567/kleanthie.georgala/odintaskgenerator:1.0.0"; hobbit:usesImage "git.project-hobbit.eu:4567/kleanthie.georgala/odinevaluationmodule:1.0.0"; hobbit:version "1.0.0"; hobbit:measuresKPI bench:averageTaskDelay ; hobbit:measuresKPI bench:microAverageRecall ; hobbit:measuresKPI bench:microAveragePrecision ; hobbit:measuresKPI bench:microAverageFmeasure ; hobbit:measuresKPI bench:macroAverageRecall ; hobbit:measuresKPI bench:macroAveragePrecision ; hobbit:measuresKPI bench:macroAverageFmeasure ; hobbit:measuresKPI bench:maxTPS ; hobbit:measuresKPI bench:averageTPS ; hobbit:measuresKPI bench:tasksRecall ; hobbit:measuresKPI bench:tasksPrecision ; hobbit:measuresKPI bench:tasksFmeasure ; hobbit:measuresKPI bench:tasksTPS ; hobbit:measuresKPI bench:tasksAnswerDelay ; hobbit:hasParameter bench:hasNumberOfDataGenerators; hobbit:hasParameter bench:hasNumberOfTaskGenerators; hobbit:hasParameter bench:hasSeed; hobbit:hasParameter bench:hasPopulation; hobbit:hasParameter bench:hasMimickingAlgorithm; hobbit:hasParameter bench:hasMimickingOutput; hobbit:hasParameter bench:hasNumberOfInsertQueries; hobbit:hasParameter bench:hasDuration. mocha:MOCHA2017-API a hobbit:API . # --- Parameters --- bench:hasNumberOfDataGenerators a hobbit:Parameter, hobbit:FeatureParameter, hobbit:ConfigurableParameter; rdfs:label "Number of data generators - agents"@en; rdfs:comment "The number of independent data generators that will insert data into the triple stores."@en; rdfs:domain hobbit:Experiment, hobbit:Challenge; rdfs:range xsd:unsignedInt; hobbit:defaultValue "2"^^xsd:unsignedInt . bench:hasNumberOfTaskGenerators a hobbit:Parameter, hobbit:FeatureParameter, hobbit:ConfigurableParameter; rdfs:label "Number of task generators - agents"@en; rdfs:comment "The number of independent task generators that perform select queries against the triple stores."@en; rdfs:domain hobbit:Experiment, hobbit:Challenge; rdfs:range xsd:unsignedInt; hobbit:defaultValue "1"^^xsd:unsignedInt . bench:hasSeed a hobbit:Parameter, hobbit:FeatureParameter, hobbit:ConfigurableParameter; rdfs:label "Seed for mimicking algorithm"@en; rdfs:comment "The seed used by the mimicking algorithm."@en; rdfs:domain hobbit:Experiment, hobbit:Challenge; rdfs:range xsd:unsignedInt; hobbit:defaultValue "100"^^xsd:unsignedInt . bench:hasPopulation a hobbit:Parameter, hobbit:FeatureParameter, hobbit:ConfigurableParameter; rdfs:label "Population of generated data"@en; rdfs:comment "Number of different events to be generated by a mimicking algorithm"@en; rdfs:domain hobbit:Experiment, hobbit:Challenge; rdfs:range xsd:unsignedInt; hobbit:defaultValue "1000"^^xsd:unsignedInt . bench:hasMimickingAlgorithm a hobbit:Parameter, hobbit:FeatureParameter, hobbit:ConfigurableParameter; rdfs:label "Name of mimicking algorithm"@en; rdfs:comment "The mimicking algorithm used by the benchmark to generate data for ingestion."@en; rdfs:domain hobbit:Experiment, hobbit:Challenge; rdfs:range xsd:string; hobbit:defaultValue "TRANSPORT_DATA"^^xsd:string. bench:hasMimickingOutput a hobbit:Parameter, hobbit:FeatureParameter, hobbit:ConfigurableParameter; rdfs:label "Name of mimicking algorithm output folder"@en; rdfs:comment "The mimicking algorithm output folder used to store the data generated from a mimicing algorithm."@en; rdfs:domain hobbit:Experiment, hobbit:Challenge; rdfs:range xsd:string; hobbit:defaultValue "output_data/"^^xsd:string. bench:hasNumberOfInsertQueries a hobbit:Parameter, hobbit:FeatureParameter, hobbit:ConfigurableParameter; rdfs:label "Number of insert queries per stream"@en; rdfs:comment "Number of insert queries performed by a data generator until a select query is performed."@en; rdfs:domain hobbit:Experiment, hobbit:Challenge; rdfs:range xsd:unsignedInt; hobbit:defaultValue "100"^^xsd:unsignedInt . bench:hasDuration a hobbit:Parameter, hobbit:FeatureParameter, hobbit:ConfigurableParameter; rdfs:label "Duration of the benchmark"@en; rdfs:comment "Duration of the benchmark"@en; rdfs:domain hobbit:Experiment, hobbit:Challenge; rdfs:range xsd:unsignedLong; hobbit:defaultValue "600000"^^xsd:unsignedLong . # --- KPIs --- bench:tasksRecall a hobbit:KPI ; rdfs:label "Evaluation of Recall for each SELECT SPARQL query"@en; rdfs:comment "Evaluation of Recall"@en; rdfs:range qb:DataSet ; hobbit:ranking hobbit:DescendingOrder . bench:tasksPrecision a hobbit:KPI ; rdfs:label "Evaluation of Precision for each SELECT SPARQL query"@en; rdfs:comment "Evaluation of Precision"@en; rdfs:range qb:DataSet ; hobbit:ranking hobbit:DescendingOrder . bench:tasksFmeasure a hobbit:KPI ; rdfs:label "Evaluation of F-measure for each SELECT SPARQL query"@en; rdfs:comment "Evaluation of Fmeasure"@en; rdfs:range qb:DataSet ; hobbit:ranking hobbit:DescendingOrder . bench:tasksTPS a hobbit:KPI ; rdfs:label "Evaluation of Triple-Per-Second for each SELECT SPARQL query"@en; rdfs:comment "Evaluation of Triples-Per-Second"@en; rdfs:range qb:DataSet ; hobbit:ranking hobbit:DescendingOrder . bench:tasksAnswerDelay a hobbit:KPI ; rdfs:label "Evaluation of task delay for each SELECT SPARQL query"@en; rdfs:comment "Evaluation of Answer Delay"@en; rdfs:range qb:DataSet ; hobbit:ranking hobbit:AscendingOrder . bench:maxTPS a hobbit:KPI; rdfs:label "Maximum Triples-Per-Seconds"@en; rdfs:comment "Max TPS achieved by the system"@en; rdfs:range xsd:double ; hobbit:ranking hobbit:DescendingOrder . bench:averageTPS a hobbit:KPI; rdfs:label "Average Triples-Per-Seconds"@en; rdfs:comment "Average TPS achieved by the system"@en; rdfs:range xsd:double ; hobbit:ranking hobbit:DescendingOrder . # --- Micro Averaging --- bench:averageTaskDelay a hobbit:KPI; rdfs:label "Average Delay of Tasks"@en; rdfs:comment "Average time need to complete a select query in seconds"@en; rdfs:range xsd:double ; hobbit:ranking hobbit:AscendingOrder . bench:microAverageRecall a hobbit:KPI; rdfs:label "Micro-Average-Recall"@en; rdfs:comment "Micro-Average-Recall = (sum of TP of all tasks) / (sum of TP of all tasks + sum of FN of all tasks)"@en; rdfs:range xsd:double ; hobbit:ranking hobbit:DescendingOrder . bench:microAveragePrecision a hobbit:KPI; rdfs:label "Micro-Average-Precision"@en; rdfs:comment "Micro-Average-Precision = (sum of TP of all tasks) / (sum of TP of all tasks + sum of FP of all tasks)"@en; rdfs:range xsd:double ; hobbit:ranking hobbit:DescendingOrder . bench:microAverageFmeasure a hobbit:KPI; rdfs:label "Micro-Average-F-Measure"@en; rdfs:comment "Micro-Average-F-Measure = (2 * Micro-Average-Precision * Micro-Average-Recall) / (Micro-Average-Precision + Micro-Average-Recall)"@en; rdfs:range xsd:double ; hobbit:ranking hobbit:DescendingOrder . # --- Macro Averaging --- bench:macroAverageRecall a hobbit:KPI; rdfs:label "Macro-Average-Recall"@en; rdfs:comment "Macro-Average-Recall = (sum of Recall of all tasks) / (number of all tasks)"@en; rdfs:range xsd:double ; hobbit:ranking hobbit:AscendingOrder . bench:macroAveragePrecision a hobbit:KPI; rdfs:label "Macro-Average-Precision"@en; rdfs:comment "Macro-Average-Precision = (sum of Precision of all tasks) / (number of all tasks)"@en; rdfs:range xsd:double ; hobbit:ranking hobbit:AscendingOrder . bench:macroAverageFmeasure a hobbit:KPI; rdfs:label "Macro-Average-F-Measure"@en; rdfs:comment "Macro-Average-F-Measure = (2 * Macro-Average-Recall * Macro-Average-Precision) / (Macro-Average-Precision + Macro-Average-Recall)"@en; rdfs:range xsd:double ; hobbit:ranking hobbit:AscendingOrder .