Skip to content

Commit f4cbb97

Browse files
simaosimao
committed
Merge remote-tracking branch 'uptane/master' into trx-master
2 parents b5bf704 + d571e8d commit f4cbb97

File tree

12 files changed

+184
-111
lines changed

12 files changed

+184
-111
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
name: Test & Build
2+
on: [push]
3+
env:
4+
SBT_OPTS: "-Dsbt.color=true -Dscala.color=true"
5+
jobs:
6+
run-tests:
7+
name: Run tests
8+
runs-on: ubuntu-latest
9+
container: uptane/ci:latest
10+
services:
11+
db:
12+
image: mariadb:10.4
13+
env:
14+
MYSQL_ROOT_PASSWORD: "root"
15+
MYSQL_DATABASE: "ota_treehub"
16+
MYSQL_USER: "treehub"
17+
MYSQL_PASSWORD: "treehub"
18+
19+
env:
20+
DB_URL: "jdbc:mariadb://db:3306/ota_treehub"
21+
22+
steps:
23+
- uses: actions/checkout@v2
24+
- name: Coursier cache
25+
uses: coursier/cache-action@v6
26+
- run: ./deploy/ci-db-setup.sh db
27+
- run: sbt ut:test

‎.github/workflows/release.yml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
name: Release
2+
on:
3+
push:
4+
branches: [master]
5+
tags: ["*"]
6+
env:
7+
SBT_OPTS: "-Dsbt.color=true -Dscala.color=true"
8+
9+
jobs:
10+
build-images:
11+
name: Build and Push Images
12+
runs-on: ubuntu-latest
13+
steps:
14+
- uses: actions/checkout@v2
15+
- name: Log in to Docker Hub
16+
uses: docker/login-action@v1
17+
with:
18+
username: ${{ secrets.DOCKERHUB_USER }}
19+
password: ${{ secrets.DOCKERHUB_TOKEN }}
20+
- uses: olafurpg/setup-scala@v11
21+
with:
22+
java-version: [email protected]
23+
- uses: coursier/cache-action@v6
24+
- run: sbt docker:publishLocal
25+
- run: docker tag advancedtelematic/treehub:$GITHUB_SHA uptane/treehub:$GITHUB_SHA
26+
- run: docker push uptane/treehub:$GITHUB_SHA

‎.gitlab-ci.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ variables:
1010
MYSQL_DATABASE: "ota_treehub"
1111
MYSQL_USER: "treehub"
1212
MYSQL_PASSWORD: "treehub"
13-
VAULT_ADDR: "http://vault.sit-ota.aws.in.here.com"
13+
# VAULT_ADDR: defined in GitLab variables
1414
# The same image have to be used by test and code coverage jobs:
1515
TEST_IMAGE: advancedtelematic/gitlab-jobs:0.2.5
1616

‎bin/find-corrupt-objects.sh‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
#!/usr/bin/env bash
2+
3+
set -eo pipefail
4+
5+
function usage {
6+
echo " usage: $0 -c <credentials.zip> -r <repo path>"
7+
echo ""
8+
echo " -c <credentials.zip> Location of your credentials.zip file"
9+
echo " -r <repo path> Path where your local OSTree repo will be"
10+
echo " created. A local repo from a previous run"
11+
echo " of this tool can be re-used only if it's"
12+
echo " from the same account."
13+
echo ""
14+
echo " Prerequisites: ostree, curl, and jq"
15+
}
16+
17+
function get_token {
18+
treehub_creds=$(unzip -qc "$1" treehub.json)
19+
auth_server=$(jq -r '.oauth2.server' <<<"$treehub_creds")
20+
treehub_server=$(jq -r '.ostree.server' <<<"$treehub_creds")
21+
client_id=$(jq -r '.oauth2.client_id' <<<"$treehub_creds")
22+
client_secret=$(jq -r '.oauth2.client_secret' <<<"$treehub_creds")
23+
curl -s -u "$client_id:$client_secret" "$auth_server"/token -d grant_type=client_credentials | jq -r .access_token
24+
}
25+
26+
while getopts ":hc:r:" opt; do
27+
case $opt in
28+
h)
29+
usage
30+
exit 0
31+
;;
32+
c)
33+
credentials_path=$OPTARG
34+
;;
35+
r)
36+
repo_name=$OPTARG
37+
;;
38+
\?)
39+
echo "Invalid option: -$OPTARG" >&2
40+
exit 1
41+
;;
42+
:)
43+
echo "Option -$OPTARG requires an argument." >&2
44+
exit 1
45+
;;
46+
esac
47+
done
48+
49+
if [ -z "$credentials_path" ] || [ -z "$repo_name" ]
50+
then
51+
usage
52+
exit 1
53+
fi
54+
55+
token=$(get_token "$credentials_path")
56+
57+
if [ -z "$token" ]
58+
then
59+
echo "Couldn't get token"
60+
exit 1
61+
fi
62+
63+
reposerver=$(unzip -qc "$credentials_path" tufrepo.url)
64+
treehub_server=$(unzip -qc "$credentials_path" treehub.json | jq -r '.ostree.server')
65+
66+
ostree_commits=$(curl -s --header "Authorization: Bearer ${token}" ${reposerver}/api/v1/user_repo/targets.json | jq -r '.signed.targets[] |select(.custom.targetFormat == "OSTREE")|.hashes.sha256' | tr '\n' ' ' )
67+
logfile="corrupt-objects-${repo_name}-$(date -uI'seconds')"
68+
69+
ostree init --mode archive --repo="${repo_name}"
70+
ostree remote add --repo="${repo_name}" --no-gpg-verify treehub ${treehub_server} 2>/dev/null || true
71+
72+
echo ""
73+
echo "Pulling commits..."
74+
echo ""
75+
ostree pull --repo="${repo_name}" --http-trusted --mirror --http-header Authorization="Bearer ${token}" treehub ${ostree_commits}
76+
77+
echo ""
78+
echo "Checking objects..."
79+
echo ""
80+
if ostree fsck --repo="${repo_name}" -a 2>"${logfile}"; then
81+
echo ""
82+
echo " No corrupt objects found!"
83+
rm ${logfile}
84+
else
85+
echo ""
86+
echo "$(cat ${logfile} | wc -l) corrupt objects found. Details logged to ${logfile}"
87+
fi
88+

‎build.sbt‎

Lines changed: 13 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,6 @@ scalaVersion := "2.12.8"
44

55
scalacOptions := Seq("-unchecked", "-deprecation", "-encoding", "utf8")
66

7-
resolvers += "ATS Releases" at "https://nexus.ota.here.com/content/repositories/releases"
8-
9-
resolvers += "ATS Snapshots" at "https://nexus.ota.here.com/content/repositories/snapshots"
10-
11-
resolvers += "commons-logging-empty" at "https://version99.qos.ch"
12-
137
def itFilter(name: String): Boolean = name endsWith "IntegrationSpec"
148

159
def unitFilter(name: String): Boolean = !itFilter(name)
@@ -28,10 +22,10 @@ lazy val root = (project in file("."))
2822
.settings(testOptions in IntegrationTest := Seq(Tests.Filter(itFilter)))
2923
.settings(sonarSettings)
3024
.settings(Seq(libraryDependencies ++= {
31-
val akkaV = "2.5.25"
32-
val akkaHttpV = "10.1.10"
25+
val akkaV = "2.6.5"
26+
val akkaHttpV = "10.1.12"
3327
val scalaTestV = "3.0.8"
34-
val libatsV = "0.3.0-109-ge12f057"
28+
val libatsV = "0.4.0-21-g0e8d408"
3529

3630
Seq(
3731
"com.typesafe.akka" %% "akka-actor" % akkaV,
@@ -45,17 +39,16 @@ lazy val root = (project in file("."))
4539
"ch.qos.logback" % "logback-classic" % "1.2.3",
4640
"org.slf4j" % "slf4j-api" % "1.7.25",
4741

48-
"com.advancedtelematic" %% "libats" % libatsV,
49-
"com.advancedtelematic" %% "libats-http" % libatsV,
50-
"com.advancedtelematic" %% "libats-http-tracing" % libatsV,
51-
"com.advancedtelematic" %% "libats-messaging" % libatsV,
52-
"com.advancedtelematic" %% "libats-messaging-datatype" % libatsV,
53-
"com.advancedtelematic" %% "libats-auth" % libatsV,
54-
"com.advancedtelematic" %% "libats-slick" % libatsV,
55-
"com.advancedtelematic" %% "libats-metrics-akka" % libatsV,
56-
"com.advancedtelematic" %% "libats-metrics-prometheus" % libatsV,
57-
"com.advancedtelematic" %% "libats-logging" % libatsV,
58-
"com.advancedtelematic" %% "libats-logging" % libatsV,
42+
"io.github.uptane" %% "libats" % libatsV,
43+
"io.github.uptane" %% "libats-http" % libatsV,
44+
"io.github.uptane" %% "libats-http-tracing" % libatsV,
45+
"io.github.uptane" %% "libats-messaging" % libatsV,
46+
"io.github.uptane" %% "libats-messaging-datatype" % libatsV,
47+
"io.github.uptane" %% "libats-slick" % libatsV,
48+
"io.github.uptane" %% "libats-metrics-akka" % libatsV,
49+
"io.github.uptane" %% "libats-metrics-prometheus" % libatsV,
50+
"io.github.uptane" %% "libats-logging" % libatsV,
51+
"io.github.uptane" %% "libats-logging" % libatsV,
5952

6053
"org.scala-lang.modules" %% "scala-async" % "0.9.6",
6154
"org.mariadb.jdbc" % "mariadb-java-client" % "1.4.4",

‎deploy/ci-db-setup.sh‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
#!/usr/bin/env bash
2+
3+
set -xeuo pipefail
4+
5+
MYSQL_COMMAND=mysql
6+
HOST=$1
7+
MYSQL=mysql
8+
MYSQLADMIN=mysqladmin
9+
10+
until $MYSQLADMIN ping --silent --protocol=TCP -h $HOST -P 3306 -u root -proot; do
11+
echo waiting for mysql; sleep 1
12+
done
13+
14+
$MYSQL -v -h $HOST -u root -proot <<EOF
15+
CREATE DATABASE IF NOT EXISTS ota_treehub;
16+
17+
CREATE USER IF NOT EXISTS 'treehub' IDENTIFIED BY 'treehub';
18+
19+
GRANT ALL PRIVILEGES ON \`ota\_treehub%\`.* TO 'treehub'@'%';
20+
FLUSH PRIVILEGES;
21+
EOF

‎src/main/scala/com/advancedtelematic/treehub/Boot.scala‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,7 @@ package com.advancedtelematic.treehub
22

33
import akka.http.scaladsl.Http
44
import akka.http.scaladsl.server.{Directives, Route}
5-
import com.advancedtelematic.libats.data.DataType.Namespace
6-
import com.advancedtelematic.libats.http.BootApp
5+
import com.advancedtelematic.libats.http.{BootApp, NamespaceDirectives}
76
import com.advancedtelematic.libats.http.LogDirectives._
87
import com.advancedtelematic.libats.http.VersionDirectives._
98
import com.advancedtelematic.libats.http.tracing.Tracing
@@ -15,11 +14,10 @@ import com.advancedtelematic.metrics.{AkkaHttpRequestMetrics, MetricsSupport}
1514
import com.advancedtelematic.treehub.client._
1615
import com.advancedtelematic.treehub.daemon.StaleObjectArchiveActor
1716
import com.advancedtelematic.treehub.delta_store.{LocalDeltaStorage, S3DeltaStorage}
18-
import com.advancedtelematic.treehub.http.{TreeHubRoutes, Http => TreeHubHttp}
17+
import com.advancedtelematic.treehub.http.TreeHubRoutes
1918
import com.advancedtelematic.treehub.object_store.{LocalFsBlobStore, ObjectStore, S3BlobStore}
2019
import com.advancedtelematic.treehub.repo_metrics.UsageMetricsRouter
2120

22-
2321
object Boot extends BootApp with Directives with Settings with VersionInfo
2422
with BootMigrations
2523
with DatabaseConfig
@@ -35,8 +33,7 @@ object Boot extends BootApp with Directives with Settings with VersionInfo
3533

3634
val deviceRegistry = new DeviceRegistryHttpClient(deviceRegistryUri, deviceRegistryMyApi)
3735

38-
val namespaceExtractor = TreeHubHttp.extractNamespace.map(_.namespace.get).map(Namespace.apply)
39-
val deviceNamespace = TreeHubHttp.deviceNamespace(deviceRegistry)
36+
val namespaceExtractor = NamespaceDirectives.defaultNamespaceExtractor
4037

4138
lazy val objectStorage = {
4239
if(useS3) {
@@ -74,7 +71,6 @@ object Boot extends BootApp with Directives with Settings with VersionInfo
7471
tracing.traceRequests { _ =>
7572
new TreeHubRoutes(
7673
namespaceExtractor,
77-
deviceNamespace,
7874
msgPublisher,
7975
objectStore,
8076
deltaStorage,

‎src/main/scala/com/advancedtelematic/treehub/http/DeviceIdDirectives.scala‎

Lines changed: 0 additions & 52 deletions
This file was deleted.

‎src/main/scala/com/advancedtelematic/treehub/http/Http.scala‎

Lines changed: 0 additions & 21 deletions
This file was deleted.

‎src/main/scala/com/advancedtelematic/treehub/http/TreeHubRoutes.scala‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ import slick.jdbc.MySQLProfile.api._
2020
import scala.concurrent.ExecutionContext
2121

2222
class TreeHubRoutes(namespaceExtractor: Directive1[Namespace],
23-
deviceNamespace: Directive1[Namespace],
2423
messageBus: MessageBusPublisher,
2524
objectStore: ObjectStore,
2625
deltaStorage: StaticDeltaStorage,
@@ -47,10 +46,7 @@ class TreeHubRoutes(namespaceExtractor: Directive1[Namespace],
4746
ErrorHandler.handleErrors {
4847
treehubExceptionHandler {
4948
pathPrefix("api" / "v2") {
50-
allRoutes(namespaceExtractor) ~
51-
pathPrefix("mydevice") {
52-
allRoutes(deviceNamespace)
53-
}
49+
allRoutes(namespaceExtractor)
5450
} ~
5551
pathPrefix("api" / "v3") {
5652
allRoutes(namespaceExtractor)

0 commit comments

Comments
 (0)