Skip to content

Commit 005511c

Browse files
velomattvelomatt
authored
Merge pull request #56 from OpenMPDK/write-optimization
Write optimization
2 parents 395cafe + 5579216 commit 005511c

File tree

216 files changed

+31882
-821
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

216 files changed

+31882
-821
lines changed

‎.gitlab-ci.yml‎

Lines changed: 18 additions & 102 deletions
Original file line numberDiff line numberDiff line change
@@ -1,111 +1,27 @@
1-
variables:
2-
BRANCH_NAME: $CI_COMMIT_BRANCH
3-
SONAR_BRANCH: -Dsonar.branch.name=$CI_COMMIT_BRANCH
4-
SONAR_UNIT_TEST_REPORT: df_out/reports/sonar_qube_ut_coverage_report.xml
5-
COBERTURA_UNIT_TEST_REPORT: df_out/reports/cobertura.xml
6-
7-
image:
8-
name: dss-build_$BRANCH_NAME
9-
10-
workflow:
11-
rules:
12-
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
13-
variables:
14-
BRANCH_NAME: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME
15-
SONAR_BRANCH: ''
16-
- if: $CI_COMMIT_BRANCH == "master" && $CI_PIPELINE_SOURCE == "push"
17-
- if: $CI_COMMIT_BRANCH =~ /^(stable|feature)\/.*/ && $CI_PIPELINE_SOURCE == "push"
1+
include:
2+
- local: .gitlab/defaults.yml
183

194
stages:
205
- build
6+
- deploy
217
- lint
228
- test
239
- scan
10+
- coverage
2411
- archive
12+
- sync
2513

26-
build dss-sdk:
27-
stage: build
28-
script:
29-
# Download build wrapper from local SonarQube
30-
- rm -rf /build-wrapper-linux-x86
31-
- wget --no-verbose --content-disposition -E -c "$SONAR_HOST_URL/static/cpp/build-wrapper-linux-x86.zip"
32-
- unzip -q build-wrapper-linux-x86.zip -d /
33-
# Disable ssl verify from docker build env
34-
- git config --global http.sslVerify false
35-
# Build client with build-wrapper
36-
- /build-wrapper-linux-x86/build-wrapper-linux-x86-64 --out-dir bw-output/ ./scripts/build_all.sh kdd-samsung-remote
37-
variables:
38-
GIT_SUBMODULE_STRATEGY: recursive
39-
artifacts:
40-
name: release build
41-
expire_in: 300 seconds
42-
paths:
43-
- bw-output
44-
- df_out
45-
- host_out
46-
47-
build target gcov and execute unit tests:
48-
stage: build
49-
script: ./target/build.sh --run-tests --with-coverage
50-
artifacts:
51-
name: gcov build
52-
expire_in: 300 seconds
53-
paths:
54-
- df_out
55-
variables:
56-
GIT_SUBMODULE_STRATEGY: recursive
57-
58-
target unit test coverage report:
59-
stage: test
60-
script:
61-
- gcovr --xml $COBERTURA_UNIT_TEST_REPORT -r target/ df_out/
62-
- sed -i -r 's/filename="/filename="target\//g' $COBERTURA_UNIT_TEST_REPORT
63-
- sed -i -r 's/path="/path="target\//g' $SONAR_UNIT_TEST_REPORT
64-
artifacts:
65-
name: target unit test coverage report
66-
expire_in: 300 seconds
67-
paths:
68-
- $SONAR_UNIT_TEST_REPORT
69-
reports:
70-
coverage_report:
71-
coverage_format: cobertura
72-
path: $COBERTURA_UNIT_TEST_REPORT
73-
needs:
74-
- build target gcov and execute unit tests
75-
76-
sonar-scanner:
77-
stage: scan
78-
script:
79-
# Download latest sonar-scanner from sonar-source
80-
- rm -rf /sonar-scanner*
81-
- wget --no-verbose --content-disposition -E -c "https://search.maven.org/remote_content?g=org.sonarsource.scanner.cli&a=sonar-scanner-cli&v=LATEST&c=linux&e=zip"
82-
- unzip -q sonar-scanner-cli-*.zip -d /
83-
# Scan with sonar-scanner
84-
- /sonar-scanner-*-linux/bin/sonar-scanner -Dsonar.qualitygate.wait=true -Dsonar.cfamily.build-wrapper-output=bw-output -Dsonar.coverageReportPaths=$SONAR_UNIT_TEST_REPORT
85-
allow_failure: true
86-
dependencies:
87-
- build dss-sdk
88-
- target unit test coverage report
89-
90-
pycodestyle:
91-
stage: lint
92-
script: git ls-files '*.py' | xargs -n1 pycodestyle --config=./scripts/pycodestyle --show-source --show-pep8
93-
needs: []
94-
95-
shellcheck:
96-
stage: lint
97-
script: git ls-files '*.sh' | xargs -n1 shellcheck
98-
needs: []
99-
100-
archive dss-sdk:
101-
stage: archive
102-
script: /stagemergeartifacts.sh
14+
deploy DSS:
15+
stage: deploy
10316
variables:
104-
DSSGLOBLIST: "nkv-sdk-*.tgz nkv-target-*.tgz"
105-
needs:
106-
- build dss-sdk
107-
artifacts:
108-
name: release artifacts
109-
paths:
110-
- df_out/nkv-target-*.tgz
111-
- host_out/nkv-sdk-bin-*.tgz
17+
BRANCH_NAME: $BRANCH_NAME
18+
UPSTREAM_REF: $CI_MERGE_REQUEST_REF_PATH
19+
trigger:
20+
include:
21+
project: dfs/dss/dss-ansible
22+
ref: master
23+
file: .gitlab-ci.yml
24+
strategy: depend
25+
resource_group: inv_$CI_PROJECT_NAME.ini
26+
rules:
27+
- !reference [.default_rules, merge_and_push]

‎.gitlab/archive.yml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
release tarballs:
2+
stage: archive
3+
script:
4+
- /stagemergeartifacts.sh
5+
variables:
6+
DSSGLOBLIST: "nkv-sdk-*.tgz nkv-target-*.tgz"
7+
needs:
8+
- build dss-sdk
9+
rules:
10+
- !reference [.default_rules, push_only]

‎.gitlab/build.yml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
build dss-sdk:
2+
stage: build
3+
before_script:
4+
# Download build wrapper from local SonarQube
5+
- rm -rf /build-wrapper-linux-x86
6+
- wget --no-verbose --content-disposition -E -c "$SONAR_HOST_URL/static/cpp/build-wrapper-linux-x86.zip"
7+
- unzip -q build-wrapper-linux-x86.zip -d /
8+
# Disable ssl verify from docker build env
9+
- git config --global http.sslVerify false
10+
script:
11+
# Build client with build-wrapper
12+
- /build-wrapper-linux-x86/build-wrapper-linux-x86-64 --out-dir bw-output/ ./scripts/build_all.sh kdd-samsung-remote
13+
variables:
14+
GIT_SUBMODULE_STRATEGY: recursive
15+
artifacts:
16+
name: release build
17+
expire_in: 300 seconds
18+
paths:
19+
- bw-output
20+
- df_out
21+
- host_out
22+
rules:
23+
- !reference [.default_rules, merge_and_push]

‎.gitlab/coverage.yml‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
# Get the Code Coverage % from SonarQube and ingest it in GitLab
2+
# This stage must run after SonarQube has executed
3+
code coverage:
4+
stage: coverage
5+
image: alpine:3.19.1
6+
before_script:
7+
- apk add curl jq
8+
script:
9+
# Scrape the SonarQube Project Key from sonar-project.properties
10+
- export PROJECT_KEY_LINE=$(grep -E "^sonar.projectKey=" sonar-project.properties)
11+
- export PROJECT_KEY=${PROJECT_KEY_LINE//sonar.projectKey=/}
12+
# Set the base URL to the SonarQube API with Project Key
13+
- export SONAR_API_URL="$SONAR_HOST_URL/api/measures/component?metricKeys=coverage&component=$PROJECT_KEY"
14+
# Conditionally add branch or pullRequest argument if this is an MR
15+
- |
16+
if [ "$CI_PIPELINE_SOURCE" == 'merge_request_event' ]; then
17+
export SONAR_API_URL="$SONAR_API_URL&pullRequest=$CI_MERGE_REQUEST_IID"
18+
else
19+
export SONAR_API_URL="$SONAR_API_URL&branch=$CI_COMMIT_REF_NAME"
20+
fi
21+
# Query the API and use jq to parse the code coverage %
22+
- export PERCENT=$(curl -s -u "$SONAR_API_TOKEN:" "$SONAR_API_URL" | jq '.component.measures[0].value')
23+
# Print the coverage string to console so that we can scrape it with GitLab
24+
- echo "SonarQube Coverage Percent ${PERCENT//\"/}"
25+
coverage: '/^SonarQube Coverage Percent (.+)$/'
26+
needs:
27+
- sonar-scanner
28+
rules:
29+
- !reference [.default_rules, merge_and_push]

‎.gitlab/defaults.yml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
include:
2+
- project: dfs/dss/dss
3+
ref: master
4+
file: .gitlab/defaults.yml
5+
- project: dfs/dss/dss
6+
ref: master
7+
file: .gitlab/sync-github.yml
8+
- .gitlab/archive.yml
9+
- .gitlab/build.yml
10+
- .gitlab/coverage.yml
11+
- .gitlab/pycodestyle.yml
12+
- .gitlab/shellcheck.yml
13+
- .gitlab/sonar.yml
14+
- .gitlab/unittest.yml
15+
16+
sync-github:
17+
variables:
18+
GITHUB_REPO: dss-sdk

‎.gitlab/pycodestyle.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pycodestyle:
2+
stage: lint
3+
script: git ls-files '*.py' | xargs -n1 pycodestyle --config=./scripts/pycodestyle --show-source --show-pep8
4+
needs: []
5+
rules:
6+
- !reference [.default_rules, merge_and_push]

‎.gitlab/shellcheck.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
include:
2+
- project: dfs/dss/dss
3+
ref: master
4+
file: .gitlab/shellcheck.yml
5+
6+
shellcheck:
7+
variables:
8+
SHELLCHECK_VER: v0.8.0

‎.gitlab/sonar.yml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
sonar-scanner:
2+
stage: scan
3+
before_script:
4+
# Download latest sonar-scanner from sonar-source
5+
- rm -rf /sonar-scanner*
6+
- wget --no-verbose --content-disposition -E -c "https://search.maven.org/remote_content?g=org.sonarsource.scanner.cli&a=sonar-scanner-cli&v=LATEST&c=linux&e=zip"
7+
- unzip -q sonar-scanner-cli-*.zip -d /
8+
script:
9+
# Scan with sonar-scanner
10+
- /sonar-scanner-*-linux/bin/sonar-scanner -Dsonar.qualitygate.wait=true -Dsonar.cfamily.build-wrapper-output=bw-output -Dsonar.coverageReportPaths=$SONAR_UNIT_TEST_REPORT
11+
allow_failure: true
12+
needs:
13+
- build dss-sdk
14+
- target unit test
15+
variables:
16+
GIT_DEPTH: 0
17+
SONAR_UNIT_TEST_REPORT: df_out/reports/sonar_qube_ut_coverage_report.xml
18+
rules:
19+
- !reference [.default_rules, merge_and_push]

‎.gitlab/unittest.yml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
target unit test:
2+
stage: test
3+
script: ./target/build.sh --run-tests --with-coverage
4+
needs: []
5+
artifacts:
6+
name: target unit test reports
7+
expire_in: 300 seconds
8+
paths:
9+
- df_out/reports
10+
reports:
11+
coverage_report:
12+
coverage_format: cobertura
13+
path: df_out/reports/cobertura.xml
14+
variables:
15+
GIT_SUBMODULE_STRATEGY: recursive
16+
rules:
17+
- !reference [.default_rules, merge_and_push]

‎buildspec/sonar-scanner.yml‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,15 +16,18 @@ phases:
1616
- aws s3 cp --recursive "$DSSS3URI/cache/dss-sdk/$GITHUB_RUN_NUMBER/build" . --only-show-errors
1717
# Download unit test coverage report(s)
1818
- aws s3 cp --recursive "$DSSS3URI/cache/dss-sdk/$GITHUB_RUN_NUMBER/unit/df_out/reports/" df_out/reports/ --only-show-errors
19-
# Correct paths of target files in coverage report
20-
- sed -i -r 's/path="/path="target\//g' $SONAR_UNIT_TEST_REPORT
2119
# replace the old CODEBUILD_SRC_DIR with the current one in bw-output
2220
- sed -i -r "s|/codebuild/output/src[^/]+/src/github.com/OpenMPDK/dss-sdk|$CODEBUILD_SRC_DIR|g" bw-output/build-wrapper-dump.json
21+
# Download the latest sonar-scanner
22+
- rm -rf /sonar-scanner*
23+
- wget --no-verbose --content-disposition -E -c "https://search.maven.org/remote_content?g=org.sonarsource.scanner.cli&a=sonar-scanner-cli&v=LATEST&c=linux&e=zip"
24+
- unzip -q sonar-scanner-cli-*.zip -d /
25+
- rm -f sonar-scanner-cli*.zip
2326
build:
2427
commands:
2528
# Run sonar-scanner and ingest coverage report(s)
2629
- |
27-
sonar-scanner \
30+
/sonar-scanner-*-linux/bin/sonar-scanner \
2831
-Dsonar.branch.name="$([[ "$GITHUB_REF_NAME" != *"/merge" ]] && echo "$GITHUB_REF_NAME")" \
2932
-Dsonar.host.url=https://sonarcloud.io \
3033
-Dsonar.pullrequest.github.summary_comment=true \

0 commit comments

Comments
 (0)