{"id":125,"date":"2024-06-26T19:17:41","date_gmt":"2024-06-26T11:17:41","guid":{"rendered":"https:\/\/abytelalala.cn\/?p=125"},"modified":"2024-10-31T20:36:46","modified_gmt":"2024-10-31T12:36:46","slug":"%e5%9f%ba%e4%ba%8eubentu%e4%b8%8adocker%ef%bc%8chbase%e7%9a%84spark%e9%83%a8%e7%bd%b2","status":"publish","type":"post","link":"https:\/\/abytelalala.cn\/index.php\/2024\/06\/26\/%e5%9f%ba%e4%ba%8eubentu%e4%b8%8adocker%ef%bc%8chbase%e7%9a%84spark%e9%83%a8%e7%bd%b2\/","title":{"rendered":"\u57fa\u4e8eubuntu\u4e0adocker\uff0chbase\u7684spark\u90e8\u7f72"},"content":{"rendered":"\n

\u5982\u679c\u6ca1\u642d\u5efahbase\uff0c\u8be6\u60c5\u8bf7\u770b\u8fd9\u7bc7https:\/\/abytelalala.cn\/index.php\/2024\/06\/26\/%e5%9f%ba%e4%ba%8eubentudocker%e6%90%ad%e5%bb%bahadoop%ef%bc%8czookeeper%ef%bc%8chbase\/<\/a><\/p>\n\n\n\n

\u5b89\u88c5\u5305<\/h2>\n\n\n\n

<\/p>\n\n\n\n

\u4e0b\u8f7d\u5b89\u88c5\u5305
scala-2.12.13.tgz
spark-3.0.0-bin-hadoop3.2.tgz
jackson-databind-2.10.1.jar
\/\/\u4e0b\u8f7d\u94fe\u63a5Central Repository: com\/fasterxml\/jackson\/core\/jackson-databind\/2.10.1 (maven.org)<\/a>
scala-2.12.13.tgz
spark-3.0.0-bin-hadoop3.2.tgz
\u62d6\u52a8\u5230mobaXterm\u7684\u865a\u62df\u673a\u9875\u9762\/home\/cust\u76ee\u5f55<\/p>\n\n\n\n

\u5728\u6bcf\u4e2a\u673a\u5668\u8282\u70b9\u4e0a\u521b\u5efa\u76ee\u5f55\uff0c\u4e5f\u5c31\u662f\u8f93\u5165\u4e0b\u9762\u4e24\u884c\u547d\u4ee4<\/p>\n\n\n\n

su -\nmkdir \/jara<\/code><\/pre>\n\n\n\n
jackson-databind-2.10.1.jar 
\u81ea\u5df1\u65b0\u5efajara\u6587\u4ef6\u5939\uff0c\u5c06\u8fd9\u4e2ajar\u62d6\u52a8\u5230mobaXterm\u7684\u865a\u62df\u673a\u9875\u9762\/home\/cust\/jara\u76ee\u5f55<\/p>\n\n\n\n
\u79fb\u52a8\u5230hadoop01\u4e0a\uff0c\u79fb\u52a8\u4e00\u6b21\u5c31\u884c\uff0c\u914d\u7f6e\u597d\u518d\u590d\u5236\u5230\u53e6\u4e24\u53f0\u5c31\u884c\u4e86<\/p>\n\n\n\n
su - \ndocker ps \/\/\u67e5\u770b\u6307\u5b9a\u7684\u5bb9\u5668id \ndocker cp \/home\/cust\/jara\/jackson-databind-2.10.1.jar <\u6307\u5b9a\u5bb9\u5668id>:\/jara\/ \/\/\u79fb\u52a8\u5230docker\u91cc\ndocker cp \/home\/cust\/scala-2.12.13.tgz <\u6307\u5b9a\u5bb9\u5668id>:\/usr\/local\/ \/\/\u79fb\u52a8\u5230docker\u91cc\ndocker cp \/home\/cust\/spark-3.0.0-bin-hadoop3.2.tgz <\u6307\u5b9a\u5bb9\u5668id>:\/usr\/local\/ \/\/\u79fb\u52a8\u5230docker\u91cc<\/code><\/pre>\n\n\n\n
\u8fdb\u5165hadoop01\u7ec8\u7aef<\/p>\n\n\n\n
docker exec -it hadoop01 bash<\/code><\/pre>\n\n\n\n
\u914d\u7f6espark<\/h2>\n\n\n\n
cd \/usr\/local\ntar -zxvf spark-3.0.0-bin-hadoop3.2.tgz\nmv spark-3.0.0-bin-hadoop3.2  \/usr\/local\/spark\ncd \/usr\/local\/spark\/conf\nmv spark-defaults.conf.template spark-defaults.conf\nmv slaves.template slaves<\/code><\/pre>\n\n\n\n
spark-defaults.conf<\/h3>\n\n\n\n
nano spark-defaults.conf\n\n        spark.driver.extraClassPath        \/jara\/jackson-databind-2.10.1.jar\n        spark.executor.extraClassPath      \/jara\/jackson-databind-2.10.1.jar<\/code><\/pre>\n\n\n\n
slaves <\/h3>\n\n\n\n
nano slaves \n\thadoop01\n\thadoop02\n\thadoop03<\/code><\/pre>\n\n\n\n
spark-env.sh<\/h3>\n\n\n\n
nano spark-env.sh\n\tSPARK_MASTER_HOST=hadoop01\n        SPARK_MASTER_PORT=7077\n\texport JAVA_HOME=\/usr\/lib\/jvm\/java-8-openjdk-amd64\n\texport HADOOP_HOME=\/usr\/local\/hadoop\/\n\texport SPARK_CLASSPATH=\/usr\/local\/hbase\/lib\/*\n\texport SCALA_HOME=\/usr\/local\/spark\/\n\texport HADOOP_CONF_DIR=\/usr\/loacl\/hadoop\/etc\/hadoop\/\n\texport SPARK_WORKER_MERMORY=8G<\/code><\/pre>\n\n\n\n
spark-config.sh<\/h3>\n\n\n\n
cd ..\/sbin\/\nnano spark-config.sh \n\texport JAVA_HOME=\/usr\/lib\/jvm\/java-8-openjdk-amd64<\/code><\/pre>\n\n\n\n
\u73af\u5883\u53d8\u91cf\u914d\u7f6e\u4ee5\u53ca\u590d\u5236<\/h2>\n\n\n\n
cd \/usr\/local\/\ntar -zxvf scala-2.12.13.tgz\nmv  scala-2.12.13  \/usr\/local\/scala\nnano \/etc\/profile\n\texport SCALA_HOME=\/usr\/local\/scala\n\texport PATH=.:$SCALA_HOME\/bin:$PATH\n\texport SPARK_HOME=\/usr\/local\/spark\n\texport PATH=$PATH:$SPARK_HOME\/bin:$SPARK_HOME\/sbin\nsource \/etc\/profile\nscp \/jara\/jackson-databind-2.10.1.jar hadoop02:\/jara\nscp \/jara\/jackson-databind-2.10.1.jar hadoop03:\/jara\nscp \/etc\/profile hadoop03:\/etc\nscp \/etc\/profile hadoop02:\/etc\nscp -r scala hadoop02:\/usr\/local\nscp -r scala hadoop03:\/usr\/local\nscp -r spark\/ hadoop02:\/usr\/local\nscp -r spark\/ hadoop03:\/usr\/local<\/code><\/pre>\n\n\n\n
\u6d4b\u8bd5<\/h2>\n\n\n\n
cd \/usr\/local\/spark\/bin\nspark-shell\n\/\/ctrl+C \u9000\u51fa\npython3 --version \/\/\u68c0\u67e5python\u81ea\u5e26\u7684\u662f\u4ec0\u4e48\u7248\u672c<\/code><\/pre>\n\n\n\n
spark-env.sh \u6587\u4ef6\u4e2d\uff0c\u8bbe\u7f6e PYSPARK_PYTHON \u73af\u5883\u53d8\u91cf\u7684\u8def\u5f84\u4e3a\u6b63\u786e\u7684 Python \u89e3\u91ca\u5668\u8def\u5f84\uff0c\u5728\u6bcf\u4e2a\u8282\u70b9\u4e0a\u90fd\u6539\u4e00\u904d<\/p>\n\n\n\n
nano spark-env.sh\n\/\/\u52a0\u5165\u5982\u4e0b\u5185\u5bb9\nexport PYSPARK_PYTHON=\/usr\/bin\/python3.8  # \u8fd9\u91cc\u7684\u7248\u672c\u4e3a Python 3.8\uff0c\u53ef\u4ee5\u5728\/usr\/bin\u91cc\u9762\u7528ls\u547d\u4ee4\u5177\u4f53\u81ea\u5df1\u770b\u770b\uff0c\u597d\u50cf\u7248\u672c\u4e0d\u80fd\u8fc7\u9ad8\uff0c3.8\u5c31\u6b63\u597d\u3002<\/code><\/pre>\n\n\n\n
\u52a0\u5165\u4e4b\u540e\u5c31\u53ef\u4ee5\u68c0\u67e5pyspark\u4e86<\/p>\n\n\n\n
pyspark<\/code><\/pre>\n\n\n\n
\u7136\u540e\u4f1a\u51fa\u73b0\u547d\u4ee4\u884c\uff0c\u5c31\u8bf4\u660e\u53ef\u4ee5\u4e86<\/p>\n\n\n\n
\u542f\u52a8\u4e00\u4e2amaster\u8282\u70b9<\/p>\n\n\n\n
start-master.sh --webui-port 8081<\/code><\/pre>\n\n\n\n
\u542f\u52a8\u4e00\u4e2aworker\u8282\u70b9<\/p>\n\n\n\n
start-slave.sh spark:\/\/192.168.1.110:7077 --webui-port 8082\n\/\/\u8fd9\u91cc\u9762\u4f60\u9700\u8981\u77e5\u9053\u8fd9\u4e2a\u4f60\u7684docker\u6620\u5c04\u5230\u670d\u52a1\u5668\u7aef\u53e3\u7684\u670d\u52a1\u5668ip\u662f\u591a\u5c11\uff0c\u66ff\u6362\u6211\u8fd9\u4e2aip\uff0c\u7aef\u53e3\u521a\u624d\u6211\u4eec\u8bbe\u7f6e\u7684\u5c31\u662f7077\u6ca1\u9519\u4e0d\u7528\u6539\u3002<\/code><\/pre>\n\n\n\n
\u5173\u95ed\u8fd9\u4fe9\u8282\u70b9<\/p>\n\n\n\n
stop-slave.sh\nstop-master.sh<\/code><\/pre>\n\n\n\n
\u6216\u8005\u76f4\u63a5\u542f\u52a8\u5168\u90e8\uff0c\u4f46\u6ce8\u610f\u8fd9\u91cc\u5df2\u7ecf\u628ahadoop\u540c\u540d\u547d\u4ee4\u914d\u7f6e\u73af\u5883\u53d8\u91cf\u4e86\uff0c\u6240\u4ee5\u8981\u4f7f\u7528\u76f8\u5173\u76ee\u5f55\u7684\u547d\u4ee4\u3002<\/p>\n\n\n\n
\/usr\/local\/spark\/sbin\/start-all.sh --webui-port 8081<\/code><\/pre>\n\n\n\n
\u6700\u540ejps\u8fdb\u7a0b\u662f\u8fd9\u4e9b<\/p>\n\n\n\n
root@hadoop01:~# jps\n2530 QuorumPeerMain\n10403 ThriftServer\n2211 NodeManager\n4579 HRegionServer\n10980 Worker\n3366 NameNode\n10810 Master\n11163 Jps\n4428 HMaster\n1870 ResourceManager\n1534 DataNode\nroot@hadoop01:~#<\/code><\/pre>\n\n\n\n
\/\/\u53ef\u4ee5\u5907\u4efd\u63d0\u4ea4\u4e2a\u955c\u50cf\n\/usr\/local\/spark\/sbin\/stop-all.sh\nhbase-daemon.sh stop thrift\nstop-hbase.sh\nzkServer.sh stop\/\/\u8fd9\u4e2a\u8981\u4e09\u53f0\u673a\u5668\u90fd\u5206\u522b\u505c\u6b62\uff0c\u522b\u7684\u53ea\u8981\u5728\u4e3b\u8282\u70b9hadoop01\u5c31\u53ef\u4ee5\u4e86\nstop-all.sh\ndocker stop hadoop01 hadoop02 hadoop03\ndocker commit hadoop01 hbase:spark \n\/\/\u6307\u5b9a\u662fspark\u7248\u672c\u7684hbase\u955c\u50cf\uff0c\u53eb\u8fd9\u4e2a\u955c\u50cf\u540d\u5b57\u662f\u56e0\u4e3a\u4e4b\u524d\u6559\u7a0b\u91cc\u662f\u8fd9\u4e2a\u955c\u50cf\uff0c\u8fd9\u91cc\u6307\u5b9a\u4e86\u90e8\u7f72\u5b8cspark\u7248\u672c\u7684<\/code><\/pre>\n","protected":false},"excerpt":{"rendered":"
\u5982\u679c\u6ca1\u642d\u5efahbase\uff0c\u8be6\u60c5\u8bf7\u770b\u8fd9\u7bc7https:\/\/abytelalala.cn\/index.php\/2024\/ […]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[8],"tags":[14],"class_list":["post-125","post","type-post","status-publish","format-standard","hentry","category-8","tag-spark"],"_links":{"self":[{"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/posts\/125","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/comments?post=125"}],"version-history":[{"count":60,"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/posts\/125\/revisions"}],"predecessor-version":[{"id":762,"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/posts\/125\/revisions\/762"}],"wp:attachment":[{"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/media?parent=125"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/categories?post=125"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/tags?post=125"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}