2023_Spark_实验三：基于IDEA开发Scala例子_Java

一、创建一个空项目，作为整个项目的基本框架

二、创建sparkstudy模块，用于学习基本的spark基础

三、创建项目结构

1、在sparkstudy模块下的pom.xml文件中加入对应的依赖，并等待依赖包下载完毕。

在pom.xml文件中加入对应的依赖



<!-- spark及scala的版本号 -->

<properties>

<scala.version>2.11</scala.version>

<spark.version>2.1.1</spark.version>

</properties>

<!-- mysql组件

<dependency>

<groupid>mysql</groupid>

<artifactid>mysql-connector-java</artifactid>

<version>5.7.22.1</version>

</dependency> 的依赖 -->

<!-- spark各个组件的依赖 -->

<dependencies>

<!-- https://mvnrepository.com/artifact/com.thoughtworks.paranamer/paranamer -->

<dependency>

<groupid>com.thoughtworks.paranamer</groupid>

<artifactid>paranamer</artifactid>

<version>2.8</version>

</dependency>

<dependency>

<groupid>org.apache.spark</groupid>

<artifactid>spark-core_${scala.version}</artifactid>

<version>${spark.version}</version>

</dependency>

<dependency>

<groupid>org.apache.spark</groupid>

<artifactid>spark-sql_${scala.version}</artifactid>

<version>${spark.version}</version>

</dependency>

<dependency>

<groupid>org.apache.spark</groupid>

<artifactid>spark-streaming_2.11</artifactid>

<version>${spark.version}</version>

</dependency>

<dependency>

<groupid>org.apache.spark</groupid>

<artifactid>spark-mllib_2.11</artifactid>

<version>2.1.1</version>

</dependency>

<dependency>

<groupid>org.apache.spark</groupid>

<artifactid>spark-streaming-kafka-0-10_2.11</artifactid>

<version>2.3.0</version>

</dependency>

<dependency>

<groupid>org.apache.spark</groupid>

<artifactid>spark-streaming-kafka-0-8_${scala.version}</artifactid>

<version>2.3.0</version>

</dependency>

<dependency>

<groupid>net.jpountz.lz4</groupid>

<artifactid>lz4</artifactid>

<version>1.3.0</version>

</dependency>

<dependency>

<groupid>mysql</groupid>

<artifactid>mysql-connector-java</artifactid>

<version>8.0.18</version>

</dependency>

<dependency>

<groupid>org.apache.flume.flume-ng-clients</groupid>

<artifactid>flume-ng-log4jappender</artifactid>

<version>1.7.0</version>

</dependency>

<!-- <dependency>-->

<!-- <groupid>org.apache.spark</groupid>-->

<!-- <artifactid>spark-streaming-flume-sink_2.10</artifactid>-->

<!-- <version>1.5.2</version>-->

<!-- </dependency>-->

<dependency>

<groupid>org.apache.spark</groupid>

<artifactid>spark-hive_2.12</artifactid>

<version>2.4.8</version>

</dependency>

</dependencies>

<!-- 配置maven打包插件及打包类型 -->

<build>

<plugins>

<plugin>

<groupid>org.apache.maven.plugins</groupid>

<artifactid>maven-compiler-plugin</artifactid>

<version>3.8.1</version>

<configuration>

<source>1.8</source>

<target>1.8</target>

</configuration>

</plugin>

<plugin>

<groupid>org.apache.maven.plugins</groupid>

<artifactid>maven-assembly-plugin</artifactid>

<configuration>

<descriptorrefs>

<descriptorref>jar-with-dependencies</descriptorref>

</descriptorrefs>

</configuration>

</plugin>

</plugins>

</build>