Browse Source

Add a script to check the license (#2552)

* Add a script to check the license

### Motivation

Check licenses in the distribution package

### Modification

1. Add a script to check licenses, and list known licenses.
1. Remove unused dep license, lombok.
1. Remove unnecessary ignored patterns in .gitignore.
1. Add missing config items to apache-rat plugin, which just checked a small part of files before.
1. Add check to GitHub Actions process

### Result

1. Newly-added dependencies should be checked and confirmed.
1. Closes #1578

* Separate rat and dependency check

* Tee dependencies to stdout

* Add more debug messages

* Check light-weight task first
1.3.2-release
kezhenxu94 4 years ago committed by gaojun2048
parent
commit
e11fda8af3
  1. 22
      .github/workflows/ci_backend.yml
  2. 125
      .gitignore
  3. 1
      dolphinscheduler-dist/release-docs/LICENSE
  4. 19
      dolphinscheduler-dist/release-docs/licenses/LICENSE-lombok.txt
  5. 2
      dolphinscheduler-ui/package.json
  6. 18
      pom.xml
  7. 39
      tools/dependencies/check-LICENSE.sh
  8. 211
      tools/dependencies/known-dependencies.txt

22
.github/workflows/ci_backend.yml

@ -56,21 +56,9 @@ jobs:
uses: actions/setup-java@v1
with:
java-version: 1.8
- name: Check license
run: ./mvnw -B apache-rat:check
- name: Compile
run: mvn -B clean compile package -Prelease -Dmaven.test.skip=true
License-check:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
# In the checkout@v2, it doesn't support git submodule. Execute the commands manually.
- name: checkout submodules
shell: bash
run: |
git submodule sync --recursive
git -c protocol.version=2 submodule update --init --force --recursive --depth=1
- name: Set up JDK 1.8
uses: actions/setup-java@v1
with:
java-version: 1.8
- name: Check
run: mvn -B apache-rat:check
run: mvn -B clean compile install -Prelease -Dmaven.test.skip=true
- name: Check dependency license
run: tools/dependencies/check-LICENSE.sh

125
.gitignore vendored

@ -4,27 +4,24 @@
.zip
.gz
.DS_Store
.idea
.idea/
.idea/*
.target
.target/
**/**/target/**
target/*
*/target
*/target/*
dist/
all-dependencies.txt
self-modules.txt
third-party-dependencies.txt
**/target/
.settings
.nbproject
.classpath
.project
*.iml
**/*.iml
*.ipr
*.iws
*.tgz
.*.swp
.vim
.tmp
node_modules
**/node_modules
npm-debug.log
.vscode
logs/*
@ -41,109 +38,9 @@ dolphinscheduler-alert/logs/
dolphinscheduler-alert/src/main/resources/alert.properties_bak
dolphinscheduler-alert/src/main/resources/logback.xml
dolphinscheduler-server/src/main/resources/logback.xml
dolphinscheduler-ui/dist
dolphinscheduler-ui/dist/
dolphinscheduler-ui/node
dolphinscheduler-ui/dist/css/common.16ac5d9.css
dolphinscheduler-ui/dist/css/home/index.b444b91.css
dolphinscheduler-ui/dist/css/login/index.5866c64.css
dolphinscheduler-ui/dist/js/0.ac94e5d.js
dolphinscheduler-ui/dist/js/0.ac94e5d.js.map
dolphinscheduler-ui/dist/js/1.0b043a3.js
dolphinscheduler-ui/dist/js/1.0b043a3.js.map
dolphinscheduler-ui/dist/js/10.1bce3dc.js
dolphinscheduler-ui/dist/js/10.1bce3dc.js.map
dolphinscheduler-ui/dist/js/11.79f04d8.js
dolphinscheduler-ui/dist/js/11.79f04d8.js.map
dolphinscheduler-ui/dist/js/12.420daa5.js
dolphinscheduler-ui/dist/js/12.420daa5.js.map
dolphinscheduler-ui/dist/js/13.e5bae1c.js
dolphinscheduler-ui/dist/js/13.e5bae1c.js.map
dolphinscheduler-ui/dist/js/14.f2a0dca.js
dolphinscheduler-ui/dist/js/14.f2a0dca.js.map
dolphinscheduler-ui/dist/js/15.45373e8.js
dolphinscheduler-ui/dist/js/15.45373e8.js.map
dolphinscheduler-ui/dist/js/16.fecb0fc.js
dolphinscheduler-ui/dist/js/16.fecb0fc.js.map
dolphinscheduler-ui/dist/js/17.84be279.js
dolphinscheduler-ui/dist/js/17.84be279.js.map
dolphinscheduler-ui/dist/js/18.307ea70.js
dolphinscheduler-ui/dist/js/18.307ea70.js.map
dolphinscheduler-ui/dist/js/19.144db9c.js
dolphinscheduler-ui/dist/js/19.144db9c.js.map
dolphinscheduler-ui/dist/js/2.8b4ef29.js
dolphinscheduler-ui/dist/js/2.8b4ef29.js.map
dolphinscheduler-ui/dist/js/20.4c527e9.js
dolphinscheduler-ui/dist/js/20.4c527e9.js.map
dolphinscheduler-ui/dist/js/21.831b2a2.js
dolphinscheduler-ui/dist/js/21.831b2a2.js.map
dolphinscheduler-ui/dist/js/22.2b4bb2a.js
dolphinscheduler-ui/dist/js/22.2b4bb2a.js.map
dolphinscheduler-ui/dist/js/23.81467ef.js
dolphinscheduler-ui/dist/js/23.81467ef.js.map
dolphinscheduler-ui/dist/js/24.54a00e4.js
dolphinscheduler-ui/dist/js/24.54a00e4.js.map
dolphinscheduler-ui/dist/js/25.8d7bd36.js
dolphinscheduler-ui/dist/js/25.8d7bd36.js.map
dolphinscheduler-ui/dist/js/26.2ec5e78.js
dolphinscheduler-ui/dist/js/26.2ec5e78.js.map
dolphinscheduler-ui/dist/js/27.3ab48c2.js
dolphinscheduler-ui/dist/js/27.3ab48c2.js.map
dolphinscheduler-ui/dist/js/28.363088a.js
dolphinscheduler-ui/dist/js/28.363088a.js.map
dolphinscheduler-ui/dist/js/29.6c5853a.js
dolphinscheduler-ui/dist/js/29.6c5853a.js.map
dolphinscheduler-ui/dist/js/3.a0edb5b.js
dolphinscheduler-ui/dist/js/3.a0edb5b.js.map
dolphinscheduler-ui/dist/js/30.940fdd3.js
dolphinscheduler-ui/dist/js/30.940fdd3.js.map
dolphinscheduler-ui/dist/js/31.168a460.js
dolphinscheduler-ui/dist/js/31.168a460.js.map
dolphinscheduler-ui/dist/js/32.8df6594.js
dolphinscheduler-ui/dist/js/32.8df6594.js.map
dolphinscheduler-ui/dist/js/33.4480bbe.js
dolphinscheduler-ui/dist/js/33.4480bbe.js.map
dolphinscheduler-ui/dist/js/34.b407fe1.js
dolphinscheduler-ui/dist/js/34.b407fe1.js.map
dolphinscheduler-ui/dist/js/35.f340b0a.js
dolphinscheduler-ui/dist/js/35.f340b0a.js.map
dolphinscheduler-ui/dist/js/36.8880c2d.js
dolphinscheduler-ui/dist/js/36.8880c2d.js.map
dolphinscheduler-ui/dist/js/37.ea2a25d.js
dolphinscheduler-ui/dist/js/37.ea2a25d.js.map
dolphinscheduler-ui/dist/js/38.98a59ee.js
dolphinscheduler-ui/dist/js/38.98a59ee.js.map
dolphinscheduler-ui/dist/js/39.a5e958a.js
dolphinscheduler-ui/dist/js/39.a5e958a.js.map
dolphinscheduler-ui/dist/js/4.4ca44db.js
dolphinscheduler-ui/dist/js/4.4ca44db.js.map
dolphinscheduler-ui/dist/js/40.e187b1e.js
dolphinscheduler-ui/dist/js/40.e187b1e.js.map
dolphinscheduler-ui/dist/js/41.0e89182.js
dolphinscheduler-ui/dist/js/41.0e89182.js.map
dolphinscheduler-ui/dist/js/42.341047c.js
dolphinscheduler-ui/dist/js/42.341047c.js.map
dolphinscheduler-ui/dist/js/43.27b8228.js
dolphinscheduler-ui/dist/js/43.27b8228.js.map
dolphinscheduler-ui/dist/js/44.e8869bc.js
dolphinscheduler-ui/dist/js/44.e8869bc.js.map
dolphinscheduler-ui/dist/js/45.8d54901.js
dolphinscheduler-ui/dist/js/45.8d54901.js.map
dolphinscheduler-ui/dist/js/5.e1ed7f3.js
dolphinscheduler-ui/dist/js/5.e1ed7f3.js.map
dolphinscheduler-ui/dist/js/6.241ba07.js
dolphinscheduler-ui/dist/js/6.241ba07.js.map
dolphinscheduler-ui/dist/js/7.ab2e297.js
dolphinscheduler-ui/dist/js/7.ab2e297.js.map
dolphinscheduler-ui/dist/js/8.83ff814.js
dolphinscheduler-ui/dist/js/8.83ff814.js.map
dolphinscheduler-ui/dist/js/9.39cb29f.js
dolphinscheduler-ui/dist/js/9.39cb29f.js.map
dolphinscheduler-ui/dist/js/common.733e342.js
dolphinscheduler-ui/dist/js/common.733e342.js.map
dolphinscheduler-ui/dist/js/home/index.78a5d12.js
dolphinscheduler-ui/dist/js/home/index.78a5d12.js.map
dolphinscheduler-ui/dist/js/login/index.291b8e3.js
dolphinscheduler-ui/dist/js/login/index.291b8e3.js.map
dolphinscheduler-ui/dist/lib/external/
/dolphinscheduler-dao/src/main/resources/dao/data_source.properties
dolphinscheduler-dao/src/main/resources/dao/data_source.properties
.mvn/wrapper/*.jar

1
dolphinscheduler-dist/release-docs/LICENSE vendored

@ -474,7 +474,6 @@ The following components are provided under a MIT 2.0 license. See project link
The text of each license is also included at licenses/LICENSE-[project].txt.
jul-to-slf4j 1.7.25: https://mvnrepository.com/artifact/org.slf4j/jul-to-slf4j/1.7.25, MIT
lombok 1.18.6: https://mvnrepository.com/artifact/org.projectlombok/lombok/1.18.6, MIT
mssql-jdbc 6.1.0.jre8: https://mvnrepository.com/artifact/com.microsoft.sqlserver/mssql-jdbc/6.1.0.jre8, MIT
slf4j-api 1.7.5: https://mvnrepository.com/artifact/org.slf4j/slf4j-api/1.7.5, MIT

19
dolphinscheduler-dist/release-docs/licenses/LICENSE-lombok.txt vendored

@ -1,19 +0,0 @@
Copyright (C) 2009-2015 The Project Lombok Authors.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

2
dolphinscheduler-ui/package.json

@ -55,7 +55,7 @@
"html-loader": "^0.5.5",
"html-webpack-plugin": "^3.2.0",
"mini-css-extract-plugin": "^0.8.2",
"node-sass": "^4.13.1",
"node-sass": "^4.14.0",
"postcss-loader": "^3.0.0",
"progress-bar-webpack-plugin": "^1.12.1",
"rimraf": "^2.6.2",

18
pom.xml

@ -774,6 +774,22 @@
<artifactId>apache-rat-plugin</artifactId>
<version>${apache.rat.version}</version>
<configuration>
<excludeSubProjects>false</excludeSubProjects>
<addDefaultLicenseMatchers>false</addDefaultLicenseMatchers>
<licenses>
<license implementation="org.apache.rat.analysis.license.SimplePatternBasedLicense">
<licenseFamilyCategory>AL20</licenseFamilyCategory>
<licenseFamilyName>Apache License, 2.0</licenseFamilyName>
<patterns>
<pattern>Licensed to the Apache Software Foundation (ASF)</pattern>
</patterns>
</license>
</licenses>
<licenseFamilies>
<licenseFamily implementation="org.apache.rat.license.SimpleLicenseFamily">
<familyName>Apache License, 2.0</familyName>
</licenseFamily>
</licenseFamilies>
<excludes>
<exclude>**/node_modules/**</exclude>
<exclude>**/node/**</exclude>
@ -792,7 +808,7 @@
<exclude>**/*.babelrc</exclude>
<exclude>**/*.eslintrc</exclude>
<exclude>**/.mvn/jvm.config</exclude>
<exclude>**/.mvn/wrapper/maven-wrapper.properties</exclude>
<exclude>**/.mvn/wrapper/**</exclude>
</excludes>
<consoleOutput>true</consoleOutput>
</configuration>

39
tools/dependencies/check-LICENSE.sh

@ -0,0 +1,39 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
mkdir dist || true
tar -zxf dolphinscheduler-dist/target/apache-dolphinscheduler*-bin.tar.gz --strip=1 -C dist
# List all modules(jars) that belong to the DolphinScheduler itself, these will be ignored when checking the dependency
# licenses
echo '=== Self modules: ' && ./mvnw --batch-mode --quiet -Dexec.executable='echo' -Dexec.args='${project.artifactId}-${project.version}.jar' exec:exec | tee self-modules.txt
echo '=== Distributed dependencies: ' && ls dist/lib | tee all-dependencies.txt
# Exclude all self modules(jars) to generate all third-party dependencies
echo '=== Third party dependencies: ' && grep -vf self-modules.txt all-dependencies.txt | tee third-party-dependencies.txt
# 1. Compare the third-party dependencies with known dependencies, expect that all third-party dependencies are KNOWN
# and the exit code of the command is 0, otherwise we should add its license to LICENSE file and add the dependency to
# known-dependencies.txt. 2. Unify the `sort` behaviour: here we'll sort them again in case that the behaviour of `sort`
# command in target OS is different from what we used to sort the file `known-dependencies.txt`, i.e. "sort the two file
# using the same command (and default arguments)"
diff -w -B -U0 <(sort < tools/dependencies/known-dependencies.txt) <(sort < third-party-dependencies.txt)

211
tools/dependencies/known-dependencies.txt

@ -0,0 +1,211 @@
HikariCP-3.2.0.jar
activation-1.1.jar
ant-1.6.5.jar
aopalliance-1.0.jar
apache-el-8.5.35.1.jar
apacheds-i18n-2.0.0-M15.jar
apacheds-kerberos-codec-2.0.0-M15.jar
api-asn1-api-1.0.0-M20.jar
api-util-1.0.0-M20.jar
asm-3.1.jar
aspectjweaver-1.9.2.jar
audience-annotations-0.5.0.jar
avro-1.7.4.jar
aws-java-sdk-1.7.4.jar
bonecp-0.8.0.RELEASE.jar
byte-buddy-1.9.10.jar
classmate-1.4.0.jar
clickhouse-jdbc-0.1.52.jar
commons-cli-1.2.jar
commons-codec-1.6.jar
commons-collections-3.2.2.jar
commons-collections4-4.1.jar
commons-compiler-3.0.12.jar
commons-compress-1.4.1.jar
commons-configuration-1.10.jar
commons-daemon-1.0.13.jar
commons-dbcp-1.4.jar
commons-email-1.5.jar
commons-httpclient-3.0.1.jar
commons-io-2.4.jar
commons-lang-2.6.jar
commons-logging-1.1.1.jar
commons-math3-3.1.1.jar
commons-net-3.1.jar
commons-pool-1.6.jar
core-3.1.1.jar
cron-utils-5.0.5.jar
curator-client-4.3.0.jar
curator-framework-4.3.0.jar
curator-recipes-4.3.0.jar
datanucleus-api-jdo-4.2.1.jar
datanucleus-core-4.1.6.jar
datanucleus-rdbms-4.1.7.jar
derby-10.14.2.0.jar
druid-1.1.14.jar
fastjson-1.2.61.jar
gson-2.8.5.jar
guava-20.0.jar
guice-3.0.jar
guice-servlet-3.0.jar
h2-1.4.200.jar
hadoop-annotations-2.7.3.jar
hadoop-auth-2.7.3.jar
hadoop-aws-2.7.3.jar
hadoop-client-2.7.3.jar
hadoop-common-2.7.3.jar
hadoop-hdfs-2.7.3.jar
hadoop-mapreduce-client-app-2.7.3.jar
hadoop-mapreduce-client-common-2.7.3.jar
hadoop-mapreduce-client-core-2.7.3.jar
hadoop-mapreduce-client-jobclient-2.7.3.jar
hadoop-mapreduce-client-shuffle-2.7.3.jar
hadoop-yarn-api-2.7.3.jar
hadoop-yarn-client-2.7.3.jar
hadoop-yarn-common-2.7.3.jar
hadoop-yarn-server-common-2.7.3.jar
hamcrest-core-1.3.jar
hibernate-validator-6.0.14.Final.jar
hive-common-2.1.0.jar
hive-jdbc-2.1.0.jar
hive-metastore-2.1.0.jar
hive-orc-2.1.0.jar
hive-serde-2.1.0.jar
hive-service-2.1.0.jar
hive-service-rpc-2.1.0.jar
hive-storage-api-2.1.0.jar
htrace-core-3.1.0-incubating.jar
httpclient-4.4.1.jar
httpcore-4.4.1.jar
httpmime-4.5.7.jar
jackson-annotations-2.9.8.jar
jackson-core-2.9.8.jar
jackson-core-asl-1.9.13.jar
jackson-databind-2.9.8.jar
jackson-datatype-jdk8-2.9.8.jar
jackson-datatype-jsr310-2.9.8.jar
jackson-jaxrs-1.9.13.jar
jackson-mapper-asl-1.9.13.jar
jackson-module-parameter-names-2.9.8.jar
jackson-xc-1.9.13.jar
jamon-runtime-2.3.1.jar
janino-3.0.12.jar
java-xmlbuilder-0.4.jar
javax.activation-api-1.2.0.jar
javax.annotation-api-1.3.2.jar
javax.inject-1.jar
javax.jdo-3.2.0-m3.jar
javax.mail-1.6.2.jar
javax.servlet-api-3.1.0.jar
javolution-5.5.1.jar
jaxb-api-2.3.1.jar
jaxb-impl-2.2.3-1.jar
jboss-logging-3.3.2.Final.jar
jdo-api-3.0.1.jar
jersey-client-1.9.jar
jersey-core-1.9.jar
jersey-guice-1.9.jar
jersey-json-1.9.jar
jersey-server-1.9.jar
jets3t-0.9.0.jar
jettison-1.1.jar
jetty-6.1.26.jar
jetty-continuation-9.4.14.v20181114.jar
jetty-http-9.4.14.v20181114.jar
jetty-io-9.4.14.v20181114.jar
jetty-security-9.4.14.v20181114.jar
jetty-server-9.4.14.v20181114.jar
jetty-servlet-9.4.14.v20181114.jar
jetty-servlets-9.4.14.v20181114.jar
jetty-util-6.1.26.jar
jetty-util-9.4.14.v20181114.jar
jetty-webapp-9.4.14.v20181114.jar
jetty-xml-9.4.14.v20181114.jar
jline-0.9.94.jar
jna-4.5.2.jar
jna-platform-4.5.2.jar
joda-time-2.10.1.jar
jpam-1.1.jar
jsch-0.1.42.jar
jsp-2.1-6.1.14.jar
jsp-api-2.1-6.1.14.jar
jsp-api-2.1.jar
jsqlparser-2.1.jar
jsr305-3.0.0.jar
jta-1.1.jar
jul-to-slf4j-1.7.25.jar
junit-4.12.jar
leveldbjni-all-1.8.jar
libfb303-0.9.3.jar
libthrift-0.9.3.jar
log4j-1.2-api-2.11.2.jar
log4j-1.2.17.jar
log4j-api-2.11.2.jar
log4j-core-2.11.2.jar
logback-classic-1.2.3.jar
logback-core-1.2.3.jar
lz4-1.3.0.jar
mapstruct-1.2.0.Final.jar
mssql-jdbc-6.1.0.jre8.jar
mybatis-3.5.2.jar
mybatis-plus-3.2.0.jar
mybatis-plus-annotation-3.2.0.jar
mybatis-plus-boot-starter-3.2.0.jar
mybatis-plus-core-3.2.0.jar
mybatis-plus-extension-3.2.0.jar
mybatis-spring-2.0.2.jar
netty-3.6.2.Final.jar
netty-all-4.1.33.Final.jar
opencsv-2.3.jar
oshi-core-3.5.0.jar
paranamer-2.3.jar
parquet-hadoop-bundle-1.8.1.jar
poi-3.17.jar
postgresql-42.1.4.jar
protobuf-java-2.5.0.jar
quartz-2.2.3.jar
quartz-jobs-2.2.3.jar
slf4j-api-1.7.5.jar
snakeyaml-1.23.jar
snappy-0.2.jar
snappy-java-1.0.4.1.jar
spring-aop-5.1.5.RELEASE.jar
spring-beans-5.1.5.RELEASE.jar
spring-boot-2.1.3.RELEASE.jar
spring-boot-autoconfigure-2.1.3.RELEASE.jar
spring-boot-starter-2.1.3.RELEASE.jar
spring-boot-starter-aop-2.1.3.RELEASE.jar
spring-boot-starter-jdbc-2.1.3.RELEASE.jar
spring-boot-starter-jetty-2.1.3.RELEASE.jar
spring-boot-starter-json-2.1.3.RELEASE.jar
spring-boot-starter-logging-2.1.3.RELEASE.jar
spring-boot-starter-web-2.1.3.RELEASE.jar
spring-context-5.1.5.RELEASE.jar
spring-core-5.1.5.RELEASE.jar
spring-expression-5.1.5.RELEASE.jar
spring-jcl-5.1.5.RELEASE.jar
spring-jdbc-5.1.5.RELEASE.jar
spring-plugin-core-1.2.0.RELEASE.jar
spring-plugin-metadata-1.2.0.RELEASE.jar
spring-tx-5.1.5.RELEASE.jar
spring-web-5.1.5.RELEASE.jar
spring-webmvc-5.1.5.RELEASE.jar
springfox-core-2.9.2.jar
springfox-schema-2.9.2.jar
springfox-spi-2.9.2.jar
springfox-spring-web-2.9.2.jar
springfox-swagger-common-2.9.2.jar
springfox-swagger-ui-2.9.2.jar
springfox-swagger2-2.9.2.jar
swagger-annotations-1.5.20.jar
swagger-bootstrap-ui-1.9.3.jar
swagger-models-1.5.20.jar
tephra-api-0.6.0.jar
threetenbp-1.3.6.jar
transaction-api-1.1.jar
validation-api-2.0.1.Final.jar
xercesImpl-2.9.1.jar
xml-apis-1.4.01.jar
xmlenc-0.52.jar
xz-1.0.jar
zookeeper-3.4.14.jar
Loading…
Cancel
Save