Merge pull request #3 from uptane/trx-master

simao · web-flow · commit 92b3e57615fd · 2021-07-14T09:54:47.000+01:00
Latest changes from trx/master
diff --git a/.gitlab-ci-toradex.yml b/.gitlab-ci-toradex.yml
@@ -0,0 +1,72 @@
+stages:
+  - test
+  - build image
+  - deploy
+
+variables:
+  DOCKER_TLS_CERTDIR: ""
+  DOCKER_HOST: tcp://localhost:2375
+  DOCKER_DRIVER: overlay2
+  MYSQL_ROOT_PASSWORD: "root"
+  MYSQL_DATABASE: "ota_treehub"
+  MYSQL_USER: "treehub"
+  MYSQL_PASSWORD: "treehub"
+  SBT_OPTS: "-sbt-launch-dir .sbt/launchers -sbt-dir .sbt -ivy .ivy2 -Dsbt.color=true -Dscala.color=true"
+  ARTIFACTORY_USERNAME: cicd-innovation
+  ARTIFACTORY_URL: https://artifactory-horw.int.toradex.com/artifactory
+  SERVICE: treehub
+
+image: artifactory-horw.int.toradex.com/ota-docker-dev-horw/tdx-inno-ci-scala:0.0.1
+
+services:
+  - name: docker:dind
+  - name: mariadb:10.4
+    alias: db
+    command:
+      - --character-set-server=utf8
+      - --collation-server=utf8_unicode_ci
+      - --max_connections=1000
+
+default:
+  cache:
+    key: treehub
+    when: always
+    paths:
+      - .ivy2/
+      - .sbt/
+
+test:
+  stage: test
+  only:
+    refs:
+      - master
+      - branches
+      - merge_requests
+  variables:
+    DB_URL: "jdbc:mariadb://db:3306/ota_treehub"
+  before_script:
+    - echo "GRANT ALL PRIVILEGES ON \`ota\_treehub%\`.* TO 'treehub'@'%'; FLUSH PRIVILEGES;" > db_user.sql
+    - until mysqladmin ping --protocol=TCP -h db -P 3306 -u root -proot; do echo waiting for mysql; sleep 1; done
+    - mysql -v -h db -u root -proot < db_user.sql
+  script:
+    - sbt -J-Xmx1G -J-XX:MaxPermSize=256m -J-XX:ReservedCodeCacheSize=128m ut:test
+
+include:
+  - project: rd/innovation/continuous-delivery/ci-container-build
+    file: gitlab-ci-templates/jfrog-sbt-docker-publish.yaml
+  - project: 'rd/platform/deploy-env'
+    ref: master
+    file: '/gitlab/deploy-core-service.yml'
+
+build docker image:
+  stage: build image
+  only:
+    refs:
+      - master
+      - merge_requests
+  extends: .jfrog-sbt-docker-publish
+  variables:
+    ARTIFACTORY_REPO: ota-docker-dev-horw
+    SERVICE_NAME: treehub
+
+
diff --git a/src/main/scala/com/advancedtelematic/treehub/http/ObjectResource.scala b/src/main/scala/com/advancedtelematic/treehub/http/ObjectResource.scala
@@ -69,6 +69,9 @@ class ObjectResource(namespace: Directive1[Namespace],
 
         complete(f)
       } ~
+      (delete & hintNamespaceStorage(ns)) {
+        complete(objectStore.deleteObject(ns, objectId).map(_ => StatusCodes.NoContent))
+      } ~
       (put & hintNamespaceStorage(ns)) {
         complete(objectStore.completeClientUpload(ns, objectId).map(_ => StatusCodes.NoContent))
       } ~
diff --git a/src/main/scala/com/advancedtelematic/treehub/object_store/BlobStore.scala b/src/main/scala/com/advancedtelematic/treehub/object_store/BlobStore.scala
@@ -1,5 +1,6 @@
 package com.advancedtelematic.treehub.object_store
 
+import akka.Done
 import akka.http.scaladsl.model._
 import akka.stream.scaladsl.Source
 import akka.util.ByteString
@@ -16,6 +17,8 @@ object BlobStore {
 }
 
 trait BlobStore {
+  def deleteObject(ns: Namespace, objectId: ObjectId): Future[Done]
+
   def storeStream(namespace: Namespace, id: ObjectId, size: Long, blob: Source[ByteString, _]): Future[Long]
 
   val supportsOutOfBandStorage: Boolean
diff --git a/src/main/scala/com/advancedtelematic/treehub/object_store/LocalFsBlobStore.scala b/src/main/scala/com/advancedtelematic/treehub/object_store/LocalFsBlobStore.scala
@@ -1,8 +1,9 @@
 package com.advancedtelematic.treehub.object_store
 
+import akka.Done
+
 import java.nio.file.StandardOpenOption.{CREATE, READ, WRITE}
 import java.nio.file.{Files, Path}
-
 import akka.http.scaladsl.model.HttpResponse
 import akka.http.scaladsl.util.FastFuture
 import akka.stream.Materializer
@@ -16,7 +17,6 @@ import org.slf4j.LoggerFactory
 import scala.concurrent.{ExecutionContext, Future}
 import scala.util.Try
 
-
 object LocalFsBlobStore {
   private val _log = LoggerFactory.getLogger(this.getClass)
 
@@ -92,4 +92,13 @@ class LocalFsBlobStore(root: Path)(implicit ec: ExecutionContext, mat: Materiali
   }
 
   override val supportsOutOfBandStorage: Boolean = false
+
+  override def deleteObject(ns: Namespace, objectId: ObjectId): Future[Done] = {
+    val f = Try {
+      Files.delete(objectId.path(namespacePath(ns)))
+      Done
+    }
+
+    Future.fromTry(f)
+  }
 }
diff --git a/src/main/scala/com/advancedtelematic/treehub/object_store/ObjectStore.scala b/src/main/scala/com/advancedtelematic/treehub/object_store/ObjectStore.scala
@@ -1,7 +1,8 @@
 package com.advancedtelematic.treehub.object_store
 
-import java.io.File
+import akka.Done
 
+import java.io.File
 import akka.http.scaladsl.model.HttpResponse
 import akka.http.scaladsl.util.FastFuture
 import akka.stream.scaladsl.{FileIO, Source}
@@ -16,6 +17,10 @@ import slick.jdbc.MySQLProfile.api._
 import scala.concurrent.{ExecutionContext, Future}
 
 class ObjectStore(blobStore: BlobStore)(implicit ec: ExecutionContext, db: Database) extends ObjectRepositorySupport {
+  def deleteObject(ns: Namespace, objectId: ObjectId): Future[Done] = for {
+    deleted <- objectRepository.delete(ns, objectId)
+    _ <- if(deleted > 0) blobStore.deleteObject(ns, objectId) else FastFuture.failed(Errors.ObjectNotFound)
+  } yield Done
 
   import scala.async.Async._
 
diff --git a/src/main/scala/com/advancedtelematic/treehub/object_store/S3BlobStore.scala b/src/main/scala/com/advancedtelematic/treehub/object_store/S3BlobStore.scala
@@ -1,11 +1,12 @@
 package com.advancedtelematic.treehub.object_store
 
+import akka.Done
+
 import java.io.File
 import java.nio.file.Paths
 import java.time.temporal.ChronoUnit
 import java.time.{Duration, Instant}
 import java.util.Date
-
 import akka.http.scaladsl.model.headers.Location
 import akka.http.scaladsl.model.{HttpResponse, StatusCodes, Uri}
 import akka.stream.Materializer
@@ -144,6 +145,14 @@ class S3BlobStore(s3Credentials: S3Credentials, s3client: AmazonS3, allowRedirec
     objectId.path(Paths.get(namespaceDir(namespace))).toString
 
   override val supportsOutOfBandStorage: Boolean = true
+
+  override def deleteObject(ns: Namespace, objectId: ObjectId): Future[Done] =
+    Future {
+      blocking {
+        s3client.deleteObject(bucketId, objectFilename(ns, objectId))
+        Done
+      }
+    }
 }
 
 object S3Client {
diff --git a/src/test/scala/com/advancedtelematic/treehub/http/ObjectResourceSpec.scala b/src/test/scala/com/advancedtelematic/treehub/http/ObjectResourceSpec.scala
@@ -7,6 +7,7 @@ package com.advancedtelematic.treehub.http
 import akka.http.scaladsl.model._
 import akka.http.scaladsl.unmarshalling.PredefinedFromEntityUnmarshallers.byteArrayUnmarshaller
 import akka.pattern.ask
+import com.advancedtelematic.libats.http.Errors.MissingEntity
 import com.advancedtelematic.treehub.db.ObjectRepositorySupport
 import com.advancedtelematic.util.FakeUsageUpdate.{CurrentBandwith, CurrentStorage}
 import com.advancedtelematic.util.ResourceSpec.ClientTObject
@@ -43,6 +44,71 @@ class ObjectResourceSpec extends TreeHubSpec with ResourceSpec with ObjectReposi
     }
   }
 
+  test("DELETE removes object") {
+    val obj = new ClientTObject()
+
+    Post(apiUri(s"objects/${obj.prefixedObjectId}"), obj.blob) ~> routes ~> check {
+      status shouldBe StatusCodes.NoContent
+    }
+
+    Delete(apiUri(s"objects/${obj.prefixedObjectId}")) ~> routes ~> check {
+      status shouldBe StatusCodes.NoContent
+    }
+
+    Get(apiUri(s"objects/${obj.prefixedObjectId}")) ~> routes ~> check {
+      status shouldBe StatusCodes.NotFound
+    }
+  }
+
+  test("DELETE removes object from database and underlying storage") {
+    val obj = new ClientTObject()
+
+    Post(apiUri(s"objects/${obj.prefixedObjectId}"), obj.blob) ~> routes ~> check {
+      status shouldBe StatusCodes.NoContent
+    }
+
+    Delete(apiUri(s"objects/${obj.prefixedObjectId}")) ~> routes ~> check {
+      status shouldBe StatusCodes.NoContent
+    }
+
+    objectRepository.find(defaultNs, obj.objectId).failed.futureValue shouldBe Errors.ObjectNotFound
+    objectStore.exists(defaultNs, obj.objectId).futureValue shouldBe false
+    localFsBlobStore.exists(defaultNs, obj.objectId).futureValue shouldBe false
+  }
+
+  test("DELETE keeps other objects intact") {
+    val obj = new ClientTObject()
+    val obj2 = new ClientTObject()
+
+    Post(apiUri(s"objects/${obj.prefixedObjectId}"), obj.blob) ~> routes ~> check {
+      status shouldBe StatusCodes.NoContent
+    }
+
+    Post(apiUri(s"objects/${obj2.prefixedObjectId}"), obj.blob) ~> routes ~> check {
+      status shouldBe StatusCodes.NoContent
+    }
+
+    Delete(apiUri(s"objects/${obj.prefixedObjectId}")) ~> routes ~> check {
+      status shouldBe StatusCodes.NoContent
+    }
+
+    Get(apiUri(s"objects/${obj.prefixedObjectId}")) ~> routes ~> check {
+      status shouldBe StatusCodes.NotFound
+    }
+
+    Get(apiUri(s"objects/${obj2.prefixedObjectId}")) ~> routes ~> check {
+      status shouldBe StatusCodes.OK
+    }
+  }
+
+  test("DELETE returns 404 when object does not exist") {
+    val obj = new ClientTObject()
+
+    Delete(apiUri(s"objects/${obj.prefixedObjectId}")) ~> routes ~> check {
+      status shouldBe StatusCodes.NotFound
+    }
+  }
+
   test("POST fails if object is empty and server does not support out of band upload") {
     val obj = new ClientTObject()
 
diff --git a/src/test/scala/com/advancedtelematic/treehub/http/TreehubRoutesSpec.scala b/src/test/scala/com/advancedtelematic/treehub/http/TreehubRoutesSpec.scala
@@ -1,5 +1,6 @@
 package com.advancedtelematic.treehub.http
 
+import akka.Done
 import akka.http.scaladsl.model.{HttpResponse, StatusCodes}
 import akka.http.scaladsl.server.Directives
 import akka.http.scaladsl.util.FastFuture
@@ -28,6 +29,8 @@ class TreehubRoutesSpec extends TreeHubSpec with ResourceSpec {
     override def readFull(namespace: DataType.Namespace, id: ObjectId): Future[ByteString] = ???
 
     override def exists(namespace: DataType.Namespace, id: ObjectId): Future[Boolean] = FastFuture.failed(new SdkClientException("Timeout on waiting"))
+
+    override def deleteObject(ns: DataType.Namespace, objectId: ObjectId): Future[Done] = FastFuture.failed(new RuntimeException("[test] delete failed"))
   }
 
   val errorObjectStore = new ObjectStore(errorBlobStore)
diff --git a/src/test/scala/com/advancedtelematic/treehub/object_store/S3BlobStoreIntegrationSpec.scala b/src/test/scala/com/advancedtelematic/treehub/object_store/S3BlobStoreIntegrationSpec.scala
@@ -56,6 +56,20 @@ class S3BlobStoreIntegrationSpec extends TreeHubSpec {
     s3BlobStore.exists(ns, obj.objectId).futureValue shouldBe true
   }
 
+  test("can delete object") {
+    val obj = new ClientTObject()
+
+    val f = async {
+      await(s3BlobStore.storeStream(ns, obj.objectId, obj.blob.size, obj.byteSource))
+    }
+
+    s3BlobStore.exists(ns, obj.objectId).futureValue shouldBe true
+
+    s3BlobStore.deleteObject(ns, obj.objectId).futureValue
+
+    s3BlobStore.exists(ns, obj.objectId).futureValue shouldBe false
+  }
+
   test("build response builds a redirect") {
     val redirectS3BlobStore = S3BlobStore(s3Credentials, allowRedirects = true)
 
diff --git a/src/test/scala/com/advancedtelematic/util/ResourceSpec.scala b/src/test/scala/com/advancedtelematic/util/ResourceSpec.scala
@@ -89,7 +89,9 @@ trait ResourceSpec extends ScalatestRouteTest with DatabaseSpec with Settings {
 
   lazy val namespaceExtractor = NamespaceDirectives.defaultNamespaceExtractor
 
-  val objectStore = new ObjectStore(new LocalFsBlobStore(Files.createTempDirectory("treehub-obj")))
+  val localFsBlobStore = new LocalFsBlobStore(Files.createTempDirectory("treehub-obj"))
+
+  val objectStore = new ObjectStore(localFsBlobStore)
 
   val deltaStore = new LocalDeltaStorage(Files.createTempDirectory("treehub-deltas"))
 
