Fixed hadoop, hadoop-aws and aws-java-sdk lib versions (#850)

* Fixed hadoop, hadoop-aws and aws-java-sdk lib versions

Author: jozefbakus

pom.xml

@@ -70,6 +70,7 @@
         <spark.version>2.4.4</spark.version>
         <hadoop.version>2.7.3</hadoop.version>
         <hadoop.scope>provided</hadoop.scope>
+        <aws-java-sdk.version>1.10.6</aws-java-sdk.version>
         <slick.version>3.3.3</slick.version>
         <tminglei.version>0.19.6</tminglei.version>
         <slick-hikaricp.version>3.3.1</slick-hikaricp.version>
@@ -111,10 +112,7 @@
         <snakeyaml.version>1.33</snakeyaml.version>
         <commons-validator.version>1.7</commons-validator.version>
         <hibernate.validator.version>6.2.0.Final</hibernate.validator.version>
-        <aws.java.sdk.bom.version>1.12.29</aws.java.sdk.bom.version>
         <absa.commons.version>1.0.3</absa.commons.version>
-        <hadoop-aws.version>2.7.3.2.6.1.0-129</hadoop-aws.version>
-        <aws-java-sdk-bundle.version>1.11.375</aws-java-sdk-bundle.version>
         <!-- Deployment -->
         <gpg.plugin.version>1.6</gpg.plugin.version>
         <nexus.staging.plugin.version>1.6.8</nexus.staging.plugin.version>
@@ -131,19 +129,6 @@
         <scm.connection><!-- defined outside the pom --></scm.connection>
         <scm.developerConnection><!-- defined outside the pom --></scm.developerConnection>
     </properties>
-
-    <dependencyManagement>
-        <dependencies>
-            <dependency>
-                <groupId>com.amazonaws</groupId>
-                <artifactId>aws-java-sdk-bom</artifactId>
-                <version>${aws.java.sdk.bom.version}</version>
-                <type>pom</type>
-                <scope>import</scope>
-            </dependency>
-        </dependencies>
-    </dependencyManagement>
-
     <dependencies>
 
         <dependency>
@@ -285,6 +270,12 @@
             </exclusions>
         </dependency>
         <!-- End Dependencies for Spark InProcessLauncher -->
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-aws</artifactId>
+            <version>${hadoop.version}</version>
+            <scope>${hadoop.scope}</scope>
+        </dependency>
 
         <dependency>
             <groupId>com.typesafe.play</groupId>
@@ -362,13 +353,25 @@
             <artifactId>hibernate-validator</artifactId>
             <version>${hibernate.validator.version}</version>
         </dependency>
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk-core</artifactId>
+            <version>${aws-java-sdk.version}</version>
+        </dependency>
         <dependency>
             <groupId>com.amazonaws</groupId>
             <artifactId>aws-java-sdk-emr</artifactId>
+            <version>${aws-java-sdk.version}</version>
         </dependency>
         <dependency>
             <groupId>com.amazonaws</groupId>
             <artifactId>aws-java-sdk-sts</artifactId>
+            <version>${aws-java-sdk.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk-s3</artifactId>
+            <version>${aws-java-sdk.version}</version>
         </dependency>
         <dependency>
             <groupId>org.springframework.security.kerberos</groupId>
@@ -444,16 +447,6 @@
             <version>${absa.commons.version}</version>
             <scope>test</scope>
         </dependency>
-        <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-aws</artifactId>
-            <version>${hadoop-aws.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>com.amazonaws</groupId>
-            <artifactId>aws-java-sdk-bundle</artifactId>
-            <version>${aws-java-sdk-bundle.version}</version>
-        </dependency>
     </dependencies>
     <build>
         <resources>

src/main/scala/za/co/absa/hyperdrive/trigger/scheduler/executors/spark/EmrClusterProviderService.scala

@@ -16,7 +16,7 @@
 
 package za.co.absa.hyperdrive.trigger.scheduler.executors.spark
 
-import com.amazonaws.services.elasticmapreduce.{AmazonElasticMapReduce, AmazonElasticMapReduceClientBuilder}
+import com.amazonaws.services.elasticmapreduce.{AmazonElasticMapReduce, AmazonElasticMapReduceClient}
 import org.springframework.stereotype.Service
 
 trait EmrClusterProviderService {
@@ -25,5 +25,5 @@ trait EmrClusterProviderService {
 
 @Service
 class EmrClusterProviderServiceImpl extends EmrClusterProviderService {
-  override def get(): AmazonElasticMapReduce = AmazonElasticMapReduceClientBuilder.standard().build()
+  override def get(): AmazonElasticMapReduce = new AmazonElasticMapReduceClient()
 }

src/main/scala/za/co/absa/hyperdrive/trigger/scheduler/executors/spark/SparkEmrClusterServiceImpl.scala

@@ -167,11 +167,11 @@ class SparkEmrClusterServiceImpl @Inject() (
         JobStatuses.Lost
       case Success(value) =>
         value match {
-          case StepState.PENDING | StepState.CANCEL_PENDING => JobStatuses.Submitting
-          case StepState.RUNNING                            => JobStatuses.Running
-          case StepState.COMPLETED                          => JobStatuses.Succeeded
-          case StepState.CANCELLED                          => JobStatuses.Killed
-          case StepState.FAILED | StepState.INTERRUPTED     => JobStatuses.Failed
+          case StepState.PENDING                        => JobStatuses.Submitting
+          case StepState.RUNNING                        => JobStatuses.Running
+          case StepState.COMPLETED                      => JobStatuses.Succeeded
+          case StepState.CANCELLED                      => JobStatuses.Killed
+          case StepState.FAILED | StepState.INTERRUPTED => JobStatuses.Failed
         }
     }
 }

src/test/scala/za/co/absa/hyperdrive/trigger/scheduler/executors/spark/SparkEmrClusterServiceTest.scala

@@ -158,7 +158,6 @@ class SparkEmrClusterServiceTest
   private val cases = Table(
     ("stepState", "jobStatus"),
     (StepState.PENDING, JobStatuses.Submitting),
-    (StepState.CANCEL_PENDING, JobStatuses.Submitting),
     (StepState.RUNNING, JobStatuses.Running),
     (StepState.COMPLETED, JobStatuses.Succeeded),
     (StepState.CANCELLED, JobStatuses.Killed),

ui/karma.conf.ci.js

@@ -21,14 +21,24 @@ const puppeteer = require('puppeteer');
 process.env.CHROMIUM_BIN = puppeteer.executablePath();
 console.log('Chromium bin path: ' + process.env.CHROMIUM_BIN);
 
+const isCI = !!process.env.GITHUB_ACTIONS;
+
 const baseConfig = require('./karma.conf.js');
 module.exports = function (config) {
   baseConfig(config);
   config.set({
     colors: false,
     autoWatch: false,
-    browsers: ['ChromiumHeadless'],
+    browsers: [isCI ? 'CustomChromeHeadless' : 'ChromiumHeadless'],
     singleRun: true,
-    restartOnFileChange: false
+    restartOnFileChange: false,
+    customLaunchers: {
+      CustomChromeHeadless: {
+        base: 'ChromeHeadless',
+        flags: [
+          ...(isCI ? ['--no-sandbox', '--disable-setuid-sandbox'] : [])
+        ]
+      }
+    }
   });
 };