Migrate to v2 of the AWS Java SDK for S3

The first part of the S3 migration.

Panda settings require an instance of an `S3BucketLoader`. Panda comes
with a default implementation for v1, but not v2, so this change creates
one.

This first requires a synchronous S3 client, which has been added to the
`AWSv2` object alongside the existing asynchronous S3 client. Creating
it here, rather than in the controller as before, allows it to be used
more widely.

Then the `getObject` method can be applied, with a `ResponseTransformer`
to retrieve the "content" of the response, i.e. the object itself.

We're removing `S3Client` as it became unused with
#27391

Co-authored-by: Roberto Tyley <[email protected]>

Author: JamieB-gu

admin/app/controllers/AdminControllers.scala

@@ -1,6 +1,6 @@
 package controllers
 import com.amazonaws.regions.Regions
-import com.amazonaws.services.s3.AmazonS3ClientBuilder
+import software.amazon.awssdk.services.s3.S3Client
 import com.softwaremill.macwire._
 import common.PekkoAsync
 import controllers.admin._
@@ -15,6 +15,7 @@ import play.api.mvc.ControllerComponents
 import services.{OphanApi, ParameterStoreService, RedirectService}
 import conf.Configuration.aws.mandatoryCredentials
 import org.apache.pekko.stream.Materializer
+import utils.AWSv2
 
 trait AdminControllers {
   def pekkoAsync: PekkoAsync
@@ -44,20 +45,12 @@ trait AdminControllers {
   def dfpApi: DfpApi
   def parameterStoreService: ParameterStoreService
 
-  private lazy val s3Client = AmazonS3ClientBuilder
-    .standard()
-    .withRegion(Regions.EU_WEST_1)
-    .withCredentials(
-      mandatoryCredentials,
-    )
-    .build()
-
   lazy val auth = new GuardianAuthWithExemptions(
     controllerComponents,
     wsClient,
     toolsDomainPrefix = "frontend",
     oauthCallbackPath = routes.GuardianAuthWithExemptions.oauthCallback.path,
-    s3Client,
+    AWSv2.S3Sync,
     system = "frontend-admin",
     extraDoNotAuthenticatePathPrefixes = Seq(
       // Date: 06 July 2021

article/app/AppLoader.scala

@@ -1,28 +1,26 @@
 import _root_.commercial.targeting.TargetingLifecycle
-import org.apache.pekko.actor.{ActorSystem => PekkoActorSystem}
 import app.{FrontendApplicationLoader, FrontendBuildInfo, FrontendComponents}
 import com.softwaremill.macwire._
 import common._
 import common.dfp.DfpAgentLifecycle
-import concurrent.BlockingOperations
-import conf.{CachedHealthCheckLifeCycle, Configuration}
+import conf.CachedHealthCheckLifeCycle
 import conf.switches.SwitchboardLifecycle
 import contentapi.{CapiHttpClient, ContentApiClient, HttpClient}
 import controllers.{ArticleControllers, HealthCheck}
 import dev.{DevAssetsController, DevParametersHttpRequestHandler}
 import http.{CommonFilters, CorsHttpErrorHandler}
 import jobs.StoreNavigationLifecycleComponent
 import model.ApplicationIdentity
+import org.apache.pekko.actor.{ActorSystem => PekkoActorSystem}
 import play.api.ApplicationLoader.Context
 import play.api.BuiltInComponentsFromContext
 import play.api.http.{HttpErrorHandler, HttpRequestHandler}
 import play.api.mvc.EssentialFilter
 import play.api.routing.Router
 import router.Routes
-import services.fronts.FrontJsonFapiLive
 import services.newsletters.{NewsletterApi, NewsletterSignupAgent, NewsletterSignupLifecycle}
 import services.ophan.SurgingContentAgentLifecycle
-import services.{NewspaperBooksAndSectionsAutoRefresh, OphanApi, S3Client, S3ClientImpl, SkimLinksCacheLifeCycle}
+import services.{NewspaperBooksAndSectionsAutoRefresh, OphanApi, SkimLinksCacheLifeCycle}
 
 class AppLoader extends FrontendApplicationLoader {
   override def buildComponents(context: Context): FrontendComponents =

article/app/controllers/ArticleControllers.scala

@@ -6,8 +6,8 @@ import model.ApplicationContext
 import play.api.libs.ws.WSClient
 import play.api.mvc.ControllerComponents
 import renderers.DotcomRenderingService
-import services.{NewsletterService, NewspaperBookSectionTagAgent, NewspaperBookTagAgent, S3Client}
 import services.newsletters.NewsletterSignupAgent
+import services.{NewsletterService, NewspaperBookSectionTagAgent, NewspaperBookTagAgent}
 
 trait ArticleControllers {
   def contentApiClient: ContentApiClient

article/app/services/S3Client.scala

@@ -1,12 +1,8 @@
 import app.FrontendComponents
-import services.S3Client
-import org.mockito.Mockito._
-import org.scalatestplus.mockito.MockitoSugar
 import play.api.ApplicationLoader.Context
 import play.api.BuiltInComponentsFromContext
 import renderers.DotcomRenderingService
-import test.WithTestContentApiClient
-import test.DCRFake
+import test.{DCRFake, WithTestContentApiClient}
 
 trait TestComponents extends WithTestContentApiClient {
   self: AppComponents =>

article/test/TestAppLoader.scala

@@ -1,7 +1,8 @@
 package http
 
 import com.amazonaws.regions.Regions
-import com.amazonaws.services.s3.AmazonS3
+import software.amazon.awssdk.core.sync.ResponseTransformer
+import software.amazon.awssdk.services.s3.S3Client
 import com.gu.pandomainauth.action.AuthActions
 import com.gu.pandomainauth.model.AuthenticatedUser
 import com.gu.pandomainauth.{PanDomain, PanDomainAuthSettingsRefresher, S3BucketLoader}
@@ -22,7 +23,7 @@ class GuardianAuthWithExemptions(
     override val wsClient: WSClient,
     toolsDomainPrefix: String,
     oauthCallbackPath: String,
-    s3Client: AmazonS3,
+    s3Client: S3Client,
     system: String,
     extraDoNotAuthenticatePathPrefixes: Seq[String],
     requiredEditorialPermissionName: String,
@@ -57,7 +58,13 @@ class GuardianAuthWithExemptions(
   override lazy val panDomainSettings = PanDomainAuthSettingsRefresher(
     domain = toolsDomainSuffix,
     system,
-    S3BucketLoader.forAwsSdkV1(s3Client, "pan-domain-auth-settings"),
+    new S3BucketLoader {
+      def inputStreamFetching(key: String) =
+        s3Client.getObject(
+          _.bucket("pan-domain-auth-settings").key(key),
+          ResponseTransformer.toInputStream(),
+        )
+    },
   )
 
   override def authCallbackUrl = s"https://$toolsDomainPrefix.$toolsDomainSuffix$oauthCallbackPath"

common/app/http/GuardianAuthWithExemptions.scala

@@ -19,7 +19,7 @@ trait S3 extends GuLogging {
 
   lazy val bucket = Configuration.aws.frontendStoreBucket
 
-  lazy val client: Option[AmazonS3] = Configuration.aws.credentials.map { credentials =>
+  lazy private val client: Option[AmazonS3] = Configuration.aws.credentials.map { credentials =>
     AmazonS3Client.builder
       .withCredentials(credentials)
       .withRegion(conf.Configuration.aws.region)
@@ -76,11 +76,6 @@ trait S3 extends GuLogging {
     put(key: String, value: String, contentType: String, PublicRead)
   }
 
-  def putPublic(key: String, file: File, contentType: String): Unit = {
-    val request = new PutObjectRequest(bucket, key, file).withCannedAcl(PublicRead)
-    client.foreach(_.putObject(request))
-  }
-
   def putPrivate(key: String, value: String, contentType: String): Unit = {
     put(key: String, value: String, contentType: String, Private)
   }
@@ -169,7 +164,6 @@ object S3FrontsApi extends S3 {
 object S3Archive extends S3 {
   override lazy val bucket: String =
     if (Configuration.environment.isNonProd) "aws-frontend-archive-code" else "aws-frontend-archive"
-  def getHtml(path: String): Option[String] = get(path)
 }
 
 object S3ArchiveOriginals extends S3 {

common/app/services/S3.scala

@@ -5,7 +5,7 @@ import software.amazon.awssdk.awscore.client.builder.AwsClientBuilder
 import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient
 import software.amazon.awssdk.regions.Region
 import software.amazon.awssdk.regions.Region.EU_WEST_1
-import software.amazon.awssdk.services.s3.{S3AsyncClient, S3AsyncClientBuilder}
+import software.amazon.awssdk.services.s3.{S3AsyncClient, S3AsyncClientBuilder, S3Client, S3ClientBuilder}
 import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider
 import software.amazon.awssdk.services.sts.model.AssumeRoleRequest
 import software.amazon.awssdk.services.sts.{StsClient, StsClientBuilder}
@@ -29,6 +29,8 @@ object AWSv2 {
 
   val S3Async: S3AsyncClient = buildS3AsyncClient(credentials)
 
+  val S3Sync: S3Client = build[S3Client, S3ClientBuilder](S3Client.builder())
+
   val STS: StsClient = build[StsClient, StsClientBuilder](StsClient.builder())
 
   def stsCredentials(devProfile: String, roleArn: String): AwsCredentialsProvider = credentialsForDevAndProd(

common/app/utils/AWSv2.scala

@@ -1,8 +1,5 @@
 import agents.MostViewedAgent
-import org.apache.pekko.actor.{ActorSystem => PekkoActorSystem}
 import app.{FrontendApplicationLoader, FrontendComponents, LifecycleComponent}
-import com.amazonaws.regions.Regions
-import com.amazonaws.services.s3.AmazonS3ClientBuilder
 import com.softwaremill.macwire._
 import commercial.controllers.CommercialControllers
 import commercial.targeting.TargetingLifecycle
@@ -17,15 +14,9 @@ import cricket.controllers.CricketControllers
 import dev.DevAssetsController
 import feed.OnwardJourneyLifecycle
 import football.controllers.FootballControllers
-import http.{
-  Filters,
-  GuardianAuthWithExemptions,
-  PreviewContentSecurityPolicyFilter,
-  PreviewNoCacheFilter,
-  RequestIdFilter,
-  routes,
-}
+import http.{routes, _}
 import model.ApplicationIdentity
+import org.apache.pekko.actor.{ActorSystem => PekkoActorSystem}
 import play.api.ApplicationLoader.Context
 import play.api.http.HttpErrorHandler
 import play.api.libs.ws.WSClient
@@ -38,7 +29,7 @@ import rugby.controllers.RugbyControllers
 import services.fronts.FrontJsonFapiDraft
 import services.newsletters.NewsletterSignupLifecycle
 import services.{ConfigAgentLifecycle, OphanApi, SkimLinksCacheLifeCycle}
-import conf.Configuration.aws.mandatoryCredentials
+import utils.AWSv2
 
 trait PreviewLifecycleComponents
     extends SportServices
@@ -102,15 +93,12 @@ trait AppComponents
     with OnwardServices
     with ApplicationsServices {
 
-  private lazy val s3Client =
-    AmazonS3ClientBuilder.standard().withRegion(Regions.EU_WEST_1).withCredentials(mandatoryCredentials).build()
-
   private lazy val auth = new GuardianAuthWithExemptions(
     controllerComponents,
     wsClient,
     toolsDomainPrefix = "preview",
     oauthCallbackPath = routes.GuardianAuthWithExemptions.oauthCallback.path,
-    s3Client,
+    AWSv2.S3Sync,
     system = "preview",
     extraDoNotAuthenticatePathPrefixes = healthCheck.healthChecks.map(_.path),
     requiredEditorialPermissionName = "preview_access",

preview/app/AppLoader.scala

@@ -18,7 +18,7 @@ object Dependencies {
   val awsDynamodb = "software.amazon.awssdk" % "dynamodb" % awsSdk2Version
   val awsEc2 = "com.amazonaws" % "aws-java-sdk-ec2" % awsVersion
   val awsKinesis = "com.amazonaws" % "aws-java-sdk-kinesis" % awsVersion
-  val awsS3 = "com.amazonaws" % "aws-java-sdk-s3" % awsVersion
+  val awsS3 = "software.amazon.awssdk" % "s3" % awsSdk2Version
   val eTagCachingS3 = "com.gu.etag-caching" %% "aws-s3-sdk-v2" % "7.0.0"
   val awsSes = "com.amazonaws" % "aws-java-sdk-ses" % awsVersion
   val awsSns = "com.amazonaws" % "aws-java-sdk-sns" % awsVersion