Skip to content

Commit d67bf4c

Browse files
tuhaihetuhaihe
committed
CI: migrate from Jenkins to GitHub Actions
* Move CI/CD pipeline from Jenkins to GitHub Actions * Split workflows into two separate files: - license-check.yml: Apache RAT and binary compliance checks - build-test.yml: MADlib build and test process * Preserve existing PostgreSQL configurations and test setup * Maintain compatibility with current test exclusion rules * Keep tool scripts in original locations for reference * Rename Jenkinsfile to Jenkinsfile.deprecated This migration enables: - Better integration with GitHub ecosystem - Improved visibility of CI/CD processes - Parallel execution of license checks and build tests - Support trigger the CI by push event or manual
1 parent d03af81 commit d67bf4c

File tree

4 files changed

+269
-2
lines changed

4 files changed

+269
-2
lines changed

‎.github/workflows/build-test.yml‎

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
name: MADlib CI Checks
19+
20+
on:
21+
push:
22+
branches: [ madlib2-master, github-actions ]
23+
pull_request:
24+
branches: [ madlib2-master, github-actions ]
25+
# Allow manual triggers for testing
26+
workflow_dispatch:
27+
28+
permissions:
29+
contents: read
30+
actions: read
31+
checks: write
32+
33+
jobs:
34+
build-and-test:
35+
name: Build and Test MADlib
36+
runs-on: ubuntu-latest
37+
38+
container:
39+
image: madlib/postgres_15:jenkins
40+
options: >-
41+
--ulimit core=-1
42+
--privileged
43+
44+
steps:
45+
- uses: actions/checkout@v4
46+
with:
47+
fetch-depth: 0
48+
persist-credentials: false
49+
set-safe-directory: true
50+
51+
- name: Configure PostgreSQL
52+
run: |
53+
cp tool/pg_hba.conf.postgres /etc/postgresql/15/main/pg_hba.conf
54+
echo " * soft nproc unlimited" > /etc/security/limits.d/postgres-limits.conf
55+
service postgresql start
56+
sleep 5
57+
# Verify PostgreSQL started successfully and show version
58+
echo "PostgreSQL version:" && su -s /bin/bash - postgres -c 'psql -c "SELECT version()"' || exit 1
59+
60+
- name: Install Dependencies
61+
run: |
62+
# Fix PostgreSQL APT repository issue (focal-pgdg is no longer available)
63+
rm -f /etc/apt/sources.list.d/pgdg.list || true
64+
65+
apt-get update
66+
apt-get install -y python3-pip openjdk-11-jre-headless
67+
# Install Python packages (pypmml requires Java to be installed first)
68+
pip install mock pandas numpy xgboost scikit-learn pyyaml pyxb-x pypmml
69+
70+
- name: Build MADlib
71+
run: |
72+
mkdir -p logs
73+
rm -rf build && mkdir -p build
74+
cd build
75+
echo "---------- CMake Configuration -----------"
76+
cmake .. 2>&1 | tee ../logs/madlib_compile.log
77+
echo "---------- Make Clean -----------"
78+
make clean 2>&1 | tee -a ../logs/madlib_compile.log
79+
echo "---------- Make Build (first pass) -----------"
80+
make -j$(nproc) 2>&1 | tee -a ../logs/madlib_compile.log
81+
echo "---------- Make Build (second pass) -----------"
82+
make -j$(nproc) 2>&1 | tee -a ../logs/madlib_compile.log
83+
echo "---------- Make Install -----------"
84+
make install 2>&1 | tee -a ../logs/madlib_compile.log
85+
echo "---------- Make Package -----------"
86+
make package 2>&1 | tee -a ../logs/madlib_compile.log
87+
chown -R postgres:postgres . || true
88+
89+
- name: Run Tests
90+
run: |
91+
export PATH=$PATH:/usr/lib/postgresql/15/bin/
92+
WORKSPACE=$(pwd)
93+
94+
echo "---------- Installing MADlib -----------"
95+
# Install MADlib as postgres user
96+
# Note: Use absolute path because 'su -' resets working directory
97+
su -s /bin/bash - postgres -c "export PATH=\$PATH:/usr/lib/postgresql/15/bin/; cd $WORKSPACE/build; $WORKSPACE/build/src/bin/madpack -s mad -p postgres -c postgres/postgres@localhost:5432/postgres install" 2>&1 | tee $WORKSPACE/logs/madlib_install.log
98+
99+
mkdir -p /tmp
100+
101+
echo "---------- Removing known problematic test files -----------"
102+
# Remove known problematic test files from BUILD directory (as done in jenkins_build.sh)
103+
# These tests are known to fail in the Docker environment
104+
rm -rf $WORKSPACE/build/src/ports/postgres/modules/deep_learning/test || true
105+
rm -rf $WORKSPACE/build/src/ports/postgres/15/modules/deep_learning/test || true
106+
rm -rf $WORKSPACE/build/src/ports/postgres/modules/linalg/test/linalg.sql_in || true
107+
rm -rf $WORKSPACE/build/src/ports/postgres/modules/prob/test/prob.sql_in || true
108+
rm -rf $WORKSPACE/build/src/ports/postgres/modules/stats/test/cox_prop_hazards.sql_in || true
109+
rm -rf $WORKSPACE/build/src/ports/postgres/modules/utilities/test/path.sql_in || true
110+
rm -rf $WORKSPACE/build/src/ports/postgres/modules/mxgboost/test/madlib_xgboost.sql_in || true
111+
rm -rf $WORKSPACE/build/src/ports/postgres/modules/kmeans/test/kmeans.sql_in || true
112+
113+
echo "---------- Running dev-check -----------"
114+
# Run dev-check and unit tests as postgres user
115+
# Note: These commands will fail the workflow if tests fail (no || true)
116+
su -s /bin/bash - postgres -c "cd $WORKSPACE/build; $WORKSPACE/build/src/bin/madpack -s mad -p postgres -c postgres/postgres@localhost:5432/postgres -d /tmp dev-check" 2>&1 | tee $WORKSPACE/logs/madlib_dev_check.log
117+
118+
echo "---------- Running unit tests -----------"
119+
su -s /bin/bash - postgres -c "cd $WORKSPACE/build; $WORKSPACE/build/src/bin/madpack -s mad -p postgres -c postgres/postgres@localhost:5432/postgres -d /tmp unit-test" 2>&1 | tee -a $WORKSPACE/logs/madlib_dev_check.log
120+
121+
- name: Process Test Results
122+
if: always()
123+
run: |
124+
echo "---------- Converting test results to JUnit format -----------"
125+
WORKSPACE=$(pwd)
126+
python3 tool/jenkins/junit_export.py . $WORKSPACE/logs/madlib_dev_check.log $WORKSPACE/logs/madlib_dev_check.xml
127+
128+
- name: Publish Test Results
129+
uses: EnricoMi/publish-unit-test-result-action@v2
130+
if: always()
131+
with:
132+
files: '${{ github.workspace }}/logs/madlib_dev_check.xml'
133+
check_name: '🧪 MADlib Test Results'
134+
comment_mode: 'always'
135+
136+
- name: Upload Build Artifacts
137+
if: always()
138+
uses: actions/upload-artifact@v4
139+
with:
140+
name: madlib-packages
141+
path: build/*.deb
142+
if-no-files-found: warn
143+
144+
- name: Upload Build Logs
145+
if: always()
146+
uses: actions/upload-artifact@v4
147+
with:
148+
name: madlib-build-logs
149+
path: logs/
150+
if-no-files-found: warn

‎.github/workflows/license-check.yml‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
name: License Check
19+
20+
on:
21+
push:
22+
branches: [ madlib2-master, github-actions ]
23+
pull_request:
24+
branches: [ madlib2-master, github-actions ]
25+
# Allow manual triggers for compliance verification
26+
workflow_dispatch:
27+
28+
jobs:
29+
rat-check:
30+
name: Apache RAT License Check
31+
runs-on: ubuntu-latest
32+
33+
steps:
34+
- uses: actions/checkout@v4
35+
with:
36+
fetch-depth: 1
37+
38+
- name: Set up JDK and Maven
39+
uses: actions/setup-java@v3
40+
with:
41+
java-version: '11'
42+
distribution: 'temurin'
43+
cache: 'maven'
44+
45+
- name: Run Apache RAT Check
46+
run: |
47+
echo "Running Apache Rat license check..."
48+
mvn apache-rat:check | tee rat-output.log
49+
50+
if grep -q "\[INFO\] BUILD FAILURE" rat-output.log; then
51+
echo "::error::Apache Rat check failed - build failure detected"
52+
exit 1
53+
fi
54+
55+
- name: Run NOTICE Year and binary file checks
56+
run: |
57+
set -o pipefail
58+
chmod +x ./tool/jenkins/rat_check.sh
59+
./tool/jenkins/rat_check.sh 2>&1 | tee license_checks.txt
60+
61+
- name: Upload Rat check results
62+
if: always()
63+
uses: actions/upload-artifact@v4
64+
with:
65+
name: rat-check-results
66+
path: |
67+
rat-output.log
68+
license_checks.txt
69+
retention-days: 7
70+
71+
- name: Generate Summary
72+
if: always()
73+
run: |
74+
# Check RAT results
75+
if [[ -f rat-output.log ]] && grep -q "\[INFO\] BUILD SUCCESS" rat-output.log; then
76+
echo "### ✅ License Check Passed" >> "$GITHUB_STEP_SUMMARY"
77+
echo "All files comply with Apache License requirements." >> "$GITHUB_STEP_SUMMARY"
78+
elif [[ -f rat-output.log ]]; then
79+
echo "### ❌ License Check Failed" >> "$GITHUB_STEP_SUMMARY"
80+
echo "" >> "$GITHUB_STEP_SUMMARY"
81+
82+
if grep -q "Files with unapproved licenses:" rat-output.log; then
83+
echo "**Files with unapproved licenses:**" >> "$GITHUB_STEP_SUMMARY"
84+
echo "" >> "$GITHUB_STEP_SUMMARY"
85+
sed -n '/Files with unapproved licenses:/,/\[INFO\] ------------------------------------------------------------------------/p' rat-output.log | \
86+
grep -v "\[INFO\] ------------------------------------------------------------------------" | \
87+
grep -v "^$" | \
88+
head -20 | \
89+
sed 's/^/- /' >> "$GITHUB_STEP_SUMMARY"
90+
echo "" >> "$GITHUB_STEP_SUMMARY"
91+
fi
92+
93+
if grep -q "Rat check: Summary over all files" rat-output.log; then
94+
echo "**Summary:**" >> "$GITHUB_STEP_SUMMARY"
95+
grep "Rat check: Summary over all files" rat-output.log | sed 's/\[INFO\] //' >> "$GITHUB_STEP_SUMMARY"
96+
fi
97+
else
98+
echo "### ⚠️ No RAT Output Log Found" >> "$GITHUB_STEP_SUMMARY"
99+
fi
100+
101+
# Check additional checks results
102+
echo "" >> "$GITHUB_STEP_SUMMARY"
103+
if [[ -f license_checks.txt ]]; then
104+
if grep -q "Error" license_checks.txt || grep -q "FAILED" license_checks.txt; then
105+
echo "### ❌ Additional Checks Failed" >> "$GITHUB_STEP_SUMMARY"
106+
echo "" >> "$GITHUB_STEP_SUMMARY"
107+
echo "**Check output:**" >> "$GITHUB_STEP_SUMMARY"
108+
echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY"
109+
cat license_checks.txt >> "$GITHUB_STEP_SUMMARY"
110+
echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY"
111+
else
112+
echo "### ✅ Additional Checks Passed" >> "$GITHUB_STEP_SUMMARY"
113+
echo "- NOTICE file year is current" >> "$GITHUB_STEP_SUMMARY"
114+
echo "- Version numbers match" >> "$GITHUB_STEP_SUMMARY"
115+
echo "- No unexpected binary files found" >> "$GITHUB_STEP_SUMMARY"
116+
fi
117+
fi

Jenkinsfile renamed to Jenkinsfile.deprecated

File renamed without changes.

‎pom.xml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
<plugin>
3232
<groupId>org.apache.rat</groupId>
3333
<artifactId>apache-rat-plugin</artifactId>
34-
<version>0.11</version>
34+
<version>0.16.1</version>
3535
<configuration>
3636
<excludes>
3737
<!-- The following comes from BSD licensed snowballstem.org project -->
@@ -692,7 +692,7 @@
692692
<plugin>
693693
<groupId>org.apache.rat</groupId>
694694
<artifactId>apache-rat-plugin</artifactId>
695-
<version>0.11</version>
695+
<version>0.16.1</version>
696696
<executions>
697697
<execution>
698698
<phase>verify</phase>

0 commit comments

Comments
 (0)