From 12c249ab47efb895ea04f14862f553f2555c9024 Mon Sep 17 00:00:00 2001 From: khadgarmage Date: Thu, 4 Jun 2020 23:13:43 +0800 Subject: [PATCH 1/2] fix ci log bug (#2901) --- .github/workflows/ci_e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_e2e.yml b/.github/workflows/ci_e2e.yml index 82c81ef4e1..68230e56fb 100644 --- a/.github/workflows/ci_e2e.yml +++ b/.github/workflows/ci_e2e.yml @@ -69,6 +69,6 @@ jobs: uses: actions/upload-artifact@v1 with: name: dslogs - path: /var/lib/docker/volumes/docker-swarm_dolphinscheduler-logs/_data + path: /var/lib/docker/volumes/dolphinscheduler-logs/_data From b21259d6005ee7e8d760ea32302d668c552463f7 Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Sat, 6 Jun 2020 17:11:20 +0800 Subject: [PATCH 2/2] add MicroBenchMark model (#2851) * add MicroBenchMark model * add log * fix propertyName * small change * add ut * format * add @Test * fix maven compile error * remove enum set method * exclusion microbench ut * small change * Use assembly instead of shade Co-authored-by: dailidong --- .github/workflows/ci_ut.yml | 2 +- dolphinscheduler-microbench/pom.xml | 101 ++++++++++++++ .../base/AbstractBaseBenchmark.java | 123 ++++++++++++++++++ .../microbench/common/EnumBenchMark.java | 112 ++++++++++++++++ pom.xml | 1 + 5 files changed, 338 insertions(+), 1 deletion(-) create mode 100644 dolphinscheduler-microbench/pom.xml create mode 100644 dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/base/AbstractBaseBenchmark.java create mode 100644 dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/EnumBenchMark.java diff --git a/.github/workflows/ci_ut.yml b/.github/workflows/ci_ut.yml index 55f1259d2b..b301cf77bd 100644 --- a/.github/workflows/ci_ut.yml +++ b/.github/workflows/ci_ut.yml @@ -78,7 +78,7 @@ jobs: -Dsonar.core.codeCoveragePlugin=jacoco -Dsonar.projectKey=apache-dolphinscheduler -Dsonar.login=e4058004bc6be89decf558ac819aa1ecbee57682 - -Dsonar.exclusions=dolphinscheduler-ui/src/**/i18n/locale/*.js + -Dsonar.exclusions=dolphinscheduler-ui/src/**/i18n/locale/*.js,dolphinscheduler-microbench/src/**/* env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} diff --git a/dolphinscheduler-microbench/pom.xml b/dolphinscheduler-microbench/pom.xml new file mode 100644 index 0000000000..6080343ed0 --- /dev/null +++ b/dolphinscheduler-microbench/pom.xml @@ -0,0 +1,101 @@ + + + + + dolphinscheduler + org.apache.dolphinscheduler + 1.2.1-SNAPSHOT + + 4.0.0 + + dolphinscheduler-microbench + jar + ${project.artifactId} + + + UTF-8 + 1.21 + 1.8 + benchmarks + + + + + + org.openjdk.jmh + jmh-core + ${jmh.version} + + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmh.version} + provided + + + + junit + junit + compile + + + + org.slf4j + slf4j-api + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + ${maven-compiler-plugin.version} + + ${javac.target} + ${javac.target} + ${javac.target} + false + + + + + org.apache.maven.plugins + maven-assembly-plugin + ${maven-assembly-plugin.version} + + + + org.openjdk.jmh.Main + + + + jar-with-dependencies + + + + + + + + + \ No newline at end of file diff --git a/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/base/AbstractBaseBenchmark.java b/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/base/AbstractBaseBenchmark.java new file mode 100644 index 0000000000..25f0ae9114 --- /dev/null +++ b/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/base/AbstractBaseBenchmark.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.microbench.base; + +import org.junit.Test; +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.results.format.ResultFormatType; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.options.ChainedOptionsBuilder; +import org.openjdk.jmh.runner.options.OptionsBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; + +/** + * BaseBenchMark + * If you need to test jmh, please extend him first + */ +@Warmup(iterations = AbstractBaseBenchmark.DEFAULT_WARMUP_ITERATIONS) +@Measurement(iterations = AbstractBaseBenchmark.DEFAULT_MEASURE_ITERATIONS) +@State(Scope.Thread) +@Fork(AbstractBaseBenchmark.DEFAULT_FORKS) +public abstract class AbstractBaseBenchmark { + + static final int DEFAULT_WARMUP_ITERATIONS = 10; + + static final int DEFAULT_MEASURE_ITERATIONS = 10; + + static final int DEFAULT_FORKS = 2; + + private static Logger logger = LoggerFactory.getLogger(AbstractBaseBenchmark.class); + + + private ChainedOptionsBuilder newOptionsBuilder() { + + String className = getClass().getSimpleName(); + + ChainedOptionsBuilder optBuilder = new OptionsBuilder() + // set benchmark ClassName + .include(className); + + if (getMeasureIterations() > 0) { + optBuilder.warmupIterations(getMeasureIterations()); + } + + if (getMeasureIterations() > 0) { + optBuilder.measurementIterations(getMeasureIterations()); + } + + if (getForks() > 0) { + optBuilder.forks(getForks()); + } + + String output = getReportDir(); + if (output != null) { + boolean writeFileStatus; + String filePath = getReportDir() + className + ".json"; + File file = new File(filePath); + + if (file.exists()) { + writeFileStatus = file.delete(); + + + } else { + writeFileStatus = file.getParentFile().mkdirs(); + try { + writeFileStatus = file.createNewFile(); + } catch (IOException e) { + logger.warn("jmh test create file error" + e); + } + } + if (writeFileStatus) { + optBuilder.resultFormat(ResultFormatType.JSON) + .result(filePath); + } + } + return optBuilder; + } + + @Test + public void run() throws Exception { + new Runner(newOptionsBuilder().build()).run(); + } + + private int getWarmupIterations() { + + String value = System.getProperty("warmupIterations"); + return null != value ? Integer.parseInt(value) : -1; + } + + private int getMeasureIterations() { + String value = System.getProperty("measureIterations"); + return null != value ? Integer.parseInt(value) : -1; + } + + private static String getReportDir() { + return System.getProperty("perfReportDir"); + } + + private static int getForks() { + String value = System.getProperty("forkCount"); + return null != value ? Integer.parseInt(value) : -1; + } + + +} + diff --git a/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/EnumBenchMark.java b/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/EnumBenchMark.java new file mode 100644 index 0000000000..dcce5368e3 --- /dev/null +++ b/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/EnumBenchMark.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.microbench.common; + + +import org.apache.dolphinscheduler.microbench.base.AbstractBaseBenchmark; +import org.openjdk.jmh.annotations.*; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + *Enum values JMH test + */ +@Warmup(iterations = 2, time = 1) +@Measurement(iterations = 4, time = 1) +@State(Scope.Benchmark) +public class EnumBenchMark extends AbstractBaseBenchmark { + + @Benchmark + public boolean simpleTest(){ + return Boolean.TRUE; + } + @Param({"101", "108", "103", "104", "105", "103"}) + private int testNum; + + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MICROSECONDS) + public void enumValuesTest() { + TestTypeEnum.oldGetNameByType(testNum); + } + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MICROSECONDS) + public void enumStaticMapTest() { + TestTypeEnum.newGetNameByType(testNum); + } + + + public enum TestTypeEnum { + + TYPE_101(101, "TYPE101"), + TYPE_102(102, "TYPE102"), + TYPE_103(103, "TYPE103"), + TYPE_104(104, "TYPE104"), + TYPE_105(105, "TYPE105"), + TYPE_106(106, "TYPE106"), + TYPE_107(107, "TYPE107"), + TYPE_108(108, "TYPE108"); + + private int code; + private String name; + + public int getCode() { + return code; + } + + + public String getName() { + return name; + } + + + TestTypeEnum(int code, String name) { + this.code = code; + this.name = name; + } + + private static final Map TEST_TYPE_MAP = new HashMap<>(); + + static { + for (TestTypeEnum testTypeEnum : TestTypeEnum.values()) { + TEST_TYPE_MAP.put(testTypeEnum.code,testTypeEnum); + } + } + + public static void newGetNameByType(int code) { + if (TEST_TYPE_MAP.containsKey(code)) { + TEST_TYPE_MAP.get(code); + return; + } + throw new IllegalArgumentException("invalid code : " + code); + } + + public static void oldGetNameByType(int code) { + for (TestTypeEnum testTypeEnum : TestTypeEnum.values()) { + if (testTypeEnum.getCode() == code) { + return; + } + } + throw new IllegalArgumentException("invalid code : " + code); + } + } + +} diff --git a/pom.xml b/pom.xml index 5dca76ce4c..06121b1321 100644 --- a/pom.xml +++ b/pom.xml @@ -1010,5 +1010,6 @@ dolphinscheduler-remote dolphinscheduler-service dolphinscheduler-plugin-api + dolphinscheduler-microbench